import torch
import torch.nn as nn
import torch.nn.functional as F
import math

class MultiHeadAttention(nn.Module):
    """多头注意力机制 - 简化版本便于理解"""
    
    def __init__(self, d_model, n_heads):
        super(MultiHeadAttention, self).__init__()
        
        # 基本参数设置
        self.d_model = d_model      # 词向量维度，比如512
        self.n_heads = n_heads      # 头的数量，比如8
        self.d_k = d_model // n_heads  # 每个头的维度，比如64
        
        # 四个"眼镜"（权重矩阵）
        self.W_q = nn.Linear(d_model, d_model)  # Query的眼镜
        self.W_k = nn.Linear(d_model, d_model)  # Key的眼镜
        self.W_v = nn.Linear(d_model, d_model)  # Value的眼镜
        self.W_o = nn.Linear(d_model, d_model)  # 最终整理的眼镜
    
    def forward(self, x):
        """
        前向传播 - 主要处理流程
        
        输入：x是句子，每个词都是d_model维的向量
        输出：每个词的新理解，还是d_model维的向量
        """
        batch_size, seq_len, d_model = x.shape
        
        # 第1步：戴上三副眼镜，生成Q、K、V
        Q = self.W_q(x)  # 通过Query眼镜看句子
        K = self.W_k(x)  # 通过Key眼镜看句子
        V = self.W_v(x)  # 通过Value眼镜看句子
        
        # 第2步：把每副眼镜分给8个专家
        # 原来：(batch_size, seq_len, 512)
        # 现在：(batch_size, 8, seq_len, 64)
        Q = self.split_heads(Q, batch_size)
        K = self.split_heads(K, batch_size)
        V = self.split_heads(V, batch_size)
        
        # 第3步：8个专家各自计算注意力
        attention_output = self.attention(Q, K, V)
        
        # 第4步：把8个专家的结果拼起来
        # 从：(batch_size, 8, seq_len, 64)
        # 到：(batch_size, seq_len, 512)
        attention_output = self.combine_heads(attention_output, batch_size)
        
        # 第5步：最终整理
        output = self.W_o(attention_output)
        
        return output
    
    def split_heads(self, x, batch_size):
        """把大向量分给多个专家"""
        # 把512维分成8个64维
        x = x.view(batch_size, -1, self.n_heads, self.d_k)
        # 调整维度顺序，方便计算
        return x.transpose(1, 2)
    
    def combine_heads(self, x, batch_size):
        """把多个专家的结果拼起来"""
        # 调整维度顺序
        x = x.transpose(1, 2)
        # 把8个64维拼成1个512维
        return x.contiguous().view(batch_size, -1, self.d_model)
    
    def attention(self, Q, K, V):
        """计算注意力 - 核心算法"""
        
        # 第1步：计算匹配度分数
        # Q和K做矩阵乘法，看看谁和谁最匹配
        scores = torch.matmul(Q, K.transpose(-2, -1))
        
        # 第2步：防止分数爆炸
        # 就像把100分制改成10分制
        scores = scores / math.sqrt(self.d_k)
        
        # 第3步：转换成百分比
        # 每个词对其他词的关注度，总和为100%
        attention_weights = F.softmax(scores, dim=-1)
        
        # 第4步：按照关注度混合内容
        # 关注度高的内容占比大，关注度低的占比小
        output = torch.matmul(attention_weights, V)
        
        return output

# 使用示例
def test_multihead_attention():
    """测试多头注意力"""
    
    # 创建一个简单的例子
    d_model = 512  # 每个词用512维向量表示
    n_heads = 8    # 8个专家头
    seq_len = 4    # 句子长度为4（比如"我爱北京天安门"）
    batch_size = 1 # 一次处理1个句子
    
    # 创建多头注意力模型
    mha = MultiHeadAttention(d_model, n_heads)
    
    # 创建输入数据（随机向量代表词语）
    x = torch.randn(batch_size, seq_len, d_model)
    
    # 计算注意力
    output = mha(x)
    
    print(f"输入形状: {x.shape}")      # [1, 4, 512]
    print(f"输出形状: {output.shape}")  # [1, 4, 512]
    print("多头注意力计算完成！")
    
    return output

# 运行测试
if __name__ == "__main__":
    result = test_multihead_attention()

import torch
import torch.nn as nn
import torch.nn.functional as F

class TransformerEncoderLayer(nn.Module):
    """Transformer编码器层 - 文本分析专家"""
    
    def __init__(self, d_model, n_heads, d_ff, dropout=0.1):
        super(TransformerEncoderLayer, self).__init__()
        
        # 第一步：词语关系分析器（多头注意力）
        self.self_attention = MultiHeadAttention(d_model, n_heads)
        
        # 第二步：深度思考器（前馈网络）
        self.feed_forward = FeedForwardNetwork(d_model, d_ff)
        
        # 数据标准化器（层归一化）
        self.norm1 = nn.LayerNorm(d_model)
        self.norm2 = nn.LayerNorm(d_model)
        
        # 防过拟合器（Dropout）
        self.dropout = nn.Dropout(dropout)
        
    def forward(self, x, mask=None):
        """
        编码器层的工作流程
        
        输入：x是句子，每个词都是d_model维的向量
        输出：每个词的更深层理解，还是d_model维的向量
        """
        
        # 第一步：分析词语关系
        # 1.1 先标准化数据
        normalized_x = self.norm1(x)
        
        # 1.2 多头注意力分析关系
        attention_result = self.self_attention(normalized_x)
        
        # 1.3 防过拟合处理
        attention_result = self.dropout(attention_result)
        
        # 1.4 残差连接：保留原始信息
        x = x + attention_result
        
        # 第二步：深度思考
        # 2.1 先标准化数据
        normalized_x = self.norm2(x)
        
        # 2.2 深度思考处理
        thinking_result = self.feed_forward(normalized_x)
        
        # 2.3 防过拟合处理
        thinking_result = self.dropout(thinking_result)
        
        # 2.4 残差连接：保留之前的理解
        x = x + thinking_result
        
        return x

class FeedForwardNetwork(nn.Module):
    """前馈网络 - 深度思考器"""
    
    def __init__(self, d_model, d_ff):
        super(FeedForwardNetwork, self).__init__()
        
        # 两层神经网络：扩展 -> 压缩
        self.expand = nn.Linear(d_model, d_ff)    # 扩展思维空间
        self.compress = nn.Linear(d_ff, d_model)  # 压缩回原始大小
        
    def forward(self, x):
        """
        深度思考过程：扩展 -> 激活 -> 压缩
        
        就像：
        1. 把问题展开到更大的思维空间（d_model -> d_ff）
        2. 进行非线性思考（ReLU激活）
        3. 把结果压缩回原始大小（d_ff -> d_model）
        """
        # 扩展到更大的思维空间
        expanded = self.expand(x)
        
        # 非线性激活（ReLU：负数变0，正数不变）
        activated = F.relu(expanded)
        
        # 压缩回原始大小
        compressed = self.compress(activated)
        
        return compressed

# 使用示例
def test_encoder_layer():
    """测试编码器层"""
    
    # 参数设置
    d_model = 512    # 词向量维度
    n_heads = 8      # 注意力头数
    d_ff = 2048      # 前馈网络中间层大小
    seq_len = 10     # 句子长度
    batch_size = 2   # 批次大小
    
    # 创建编码器层
    encoder_layer = TransformerEncoderLayer(d_model, n_heads, d_ff)
    
    # 创建输入数据（模拟句子）
    x = torch.randn(batch_size, seq_len, d_model)
    
    # 通过编码器层处理
    output = encoder_layer(x)
    
    print(f"输入形状: {x.shape}")      # [2, 10, 512]
    print(f"输出形状: {output.shape}")  # [2, 10, 512]
    print("编码器层处理完成！")
    
    return output

# 运行测试
if __name__ == "__main__":
    result = test_encoder_layer()

import torch
import torch.nn.functional as F

def create_look_ahead_mask(seq_len):
    """
    创建前瞻掩码 - 防止偷看未来
    
    就像给每个位置戴上眼罩，只能看到自己和之前的位置
    """
    # 创建一个下三角矩阵（1表示可以看，0表示不能看）
    mask = torch.tril(torch.ones(seq_len, seq_len))
    return mask

def demonstrate_mask():
    """演示掩码的作用"""
    
    # 假设我们在翻译 "Hello World" -> "你好 世界"
    seq_len = 4  # 假设目标句子长度为4
    
    print("=== 掩码演示 ===")
    print("翻译进度：[你, 好, ___, ___]")
    print("位置编号：[0,  1,  2,   3]")
    
    # 创建掩码
    mask = create_look_ahead_mask(seq_len)
    print(f"\n掩码矩阵（1=可以看，0=不能看）:")
    print(mask.int())
    
    # 模拟注意力分数
    print(f"\n=== 注意力计算演示 ===")
    
    # 假设的注意力分数（随机生成）
    attention_scores = torch.randn(seq_len, seq_len)
    print("原始注意力分数:")
    print(attention_scores.round(decimals=2))
    
    # 应用掩码：不能看的位置设为很小的负数
    masked_scores = attention_scores.masked_fill(mask == 0, -1e9)
    print("\n应用掩码后的分数:")
    print(masked_scores)
    
    # 转换为概率
    attention_weights = F.softmax(masked_scores, dim=-1)
    print("\n最终注意力权重:")
    print(attention_weights.round(decimals=3))
    
    print("\n=== 解释 ===")
    print("位置0（'你'）：只关注自己")
    print("位置1（'好'）：关注'你'和自己")  
    print("位置2（正在翻译）：关注'你'、'好'和自己")
    print("位置3（未来位置）：关注前面所有位置")

# 运行演示
if __name__ == "__main__":
    demonstrate_mask()

import torch
import torch.nn as nn
import math

class PositionalEncoding(nn.Module):
    """位置编码 - 给每个词分配座位号"""
    
    def __init__(self, d_model, max_len=5000):
        super(PositionalEncoding, self).__init__()
        
        # 创建位置编码表（就像座位表）
        pe = torch.zeros(max_len, d_model)
        
        # 位置索引：0, 1, 2, 3, ...
        position = torch.arange(0, max_len).unsqueeze(1).float()
        
        # 计算不同频率（就像不同的波长）
        # 频率从高到低，让不同维度有不同的变化速度
        div_term = torch.exp(torch.arange(0, d_model, 2).float() * 
                           -(math.log(10000.0) / d_model))
        
        # 偶数维度用sin，奇数维度用cos
        pe[:, 0::2] = torch.sin(position * div_term)  # 0, 2, 4, 6, ...
        pe[:, 1::2] = torch.cos(position * div_term)  # 1, 3, 5, 7, ...
        
        # 添加batch维度，注册为buffer（不需要训练的参数）
        pe = pe.unsqueeze(0)
        self.register_buffer('pe', pe)
        
    def forward(self, x):
        """
        给词嵌入加上位置信息
        
        就像：词的含义 + 词的位置 = 完整的词表示
        """
        seq_len = x.size(1)
        return x + self.pe[:, :seq_len]

# 简单演示位置编码
def demonstrate_positional_encoding():
    """演示位置编码的效果"""
    
    print("=== 位置编码演示 ===")
    
    # 创建位置编码器
    d_model = 8  # 简化为8维，便于观察
    pe_layer = PositionalEncoding(d_model, max_len=10)
    
    # 获取前5个位置的编码
    positions = pe_layer.pe.squeeze(0)[:5]  # (5, 8)
    
    print("前5个位置的编码（简化显示）:")
    for i, pos_encoding in enumerate(positions):
        print(f"位置{i}: {pos_encoding.round(decimals=3).tolist()}")
    
    print("\n=== 观察规律 ===")
    print("1. 每个位置的编码都不同（唯一性）")
    print("2. 相邻位置的编码相似（平滑性）")
    print("3. 编码值在-1到1之间变化（有界性）")
    
    # 模拟词嵌入
    print(f"\n=== 词嵌入 + 位置编码 ===")
    
    # 假设的词嵌入（随机生成）
    word_embeddings = torch.randn(1, 3, d_model)  # 3个词
    print("原始词嵌入:")
    for i in range(3):
        print(f"词{i}: {word_embeddings[0, i].round(decimals=2).tolist()}")
    
    # 添加位置编码
    final_embeddings = pe_layer(word_embeddings)
    print("\n添加位置编码后:")
    for i in range(3):
        print(f"词{i}: {final_embeddings[0, i].round(decimals=2).tolist()}")
    
    print("\n现在每个词不仅有自己的含义，还知道自己在句子中的位置！")

# 可视化位置编码模式
def simple_visualization():
    """简单可视化位置编码"""
    
    d_model = 16
    max_len = 20
    
    pe_layer = PositionalEncoding(d_model, max_len)
    pe = pe_layer.pe.squeeze(0)  # (max_len, d_model)
    
    print("=== 位置编码模式可视化 ===")
    print("每行代表一个位置，每列代表一个维度")
    print("数值范围：-1.0 到 1.0")
    print()
    
    # 显示前10个位置的前8个维度
    for pos in range(10):
        values = pe[pos, :8]
        # 用符号表示数值大小
        symbols = []
        for val in values:
            if val > 0.5:
                symbols.append("++")
            elif val > 0:
                symbols.append(" +")
            elif val > -0.5:
                symbols.append(" -")
            else:
                symbols.append("--")
        print(f"位置{pos:2d}: {' '.join(symbols)}")
    
    print("\n图例：++ (>0.5)  + (0~0.5)  - (-0.5~0)  -- (<-0.5)")

# 运行演示
if __name__ == "__main__":
    demonstrate_positional_encoding()
    print("\n" + "="*50 + "\n")
    simple_visualization()

class EncoderLayer(nn.Module):
    def __init__(self, d_model, n_heads, d_ff, dropout=0.1):
        super().__init__()
        # 多头自注意力
        self.self_attention = MultiHeadAttention(d_model, n_heads)
        # 前馈神经网络
        self.feed_forward = FeedForward(d_model, d_ff)
        # 层归一化
        self.norm1 = nn.LayerNorm(d_model)
        self.norm2 = nn.LayerNorm(d_model)
        # Dropout
        self.dropout = nn.Dropout(dropout)
        
    def forward(self, x, mask=None):
        # 自注意力 + 残差连接 + 层归一化
        attn_output = self.self_attention(x, mask)
        x = self.norm1(x + self.dropout(attn_output))
        
        # 前馈网络 + 残差连接 + 层归一化
        ff_output = self.feed_forward(x)
        x = self.norm2(x + self.dropout(ff_output))
        
        return x

class FeedForward(nn.Module):
    def __init__(self, d_model, d_ff):
        super().__init__()
        self.linear1 = nn.Linear(d_model, d_ff)
        self.linear2 = nn.Linear(d_ff, d_model)
        
    def forward(self, x):
        return self.linear2(F.relu(self.linear1(x)))

class Transformer(nn.Module):
    """完整的Transformer模型"""
    
    def __init__(self, 
                 src_vocab_size,    # 源词汇表大小
                 tgt_vocab_size,    # 目标词汇表大小
                 d_model=512,       # 模型维度
                 n_heads=8,         # 注意力头数
                 n_layers=6,        # 层数
                 d_ff=2048,         # 前馈网络维度
                 max_len=5000,      # 最大序列长度
                 dropout=0.1):      # Dropout率
        
        super(Transformer, self).__init__()
        
        self.d_model = d_model
        
        # 嵌入层
        self.src_embedding = nn.Embedding(src_vocab_size, d_model)
        self.tgt_embedding = nn.Embedding(tgt_vocab_size, d_model)
        
        # 位置编码
        self.pos_encoding = PositionalEncoding(d_model, max_len)
        
        # 编码器栈
        self.encoder_layers = nn.ModuleList([
            TransformerEncoderLayer(d_model, n_heads, d_ff, dropout)
            for _ in range(n_layers)
        ])
        
        # 解码器栈
        self.decoder_layers = nn.ModuleList([
            TransformerDecoderLayer(d_model, n_heads, d_ff, dropout)
            for _ in range(n_layers)
        ])
        
        # 输出投影
        self.output_projection = nn.Linear(d_model, tgt_vocab_size)
        
        # Dropout
        self.dropout = nn.Dropout(dropout)
        
        # 初始化参数
        self._init_parameters()
    
    def _init_parameters(self):
        """初始化模型参数"""
        for p in self.parameters():
            if p.dim() > 1:
                nn.init.xavier_uniform_(p)
    
    def encode(self, src, src_mask=None):
        """编码器前向传播"""
        # 词嵌入 + 位置编码
        src_emb = self.src_embedding(src) * math.sqrt(self.d_model)
        src_emb = self.pos_encoding(src_emb)
        src_emb = self.dropout(src_emb)
        
        # 通过编码器层
        encoder_output = src_emb
        for layer in self.encoder_layers:
            encoder_output = layer(encoder_output, src_mask)
        
        return encoder_output
    
    def decode(self, tgt, encoder_output, tgt_mask=None, src_mask=None):
        """解码器前向传播"""
        # 词嵌入 + 位置编码
        tgt_emb = self.tgt_embedding(tgt) * math.sqrt(self.d_model)
        tgt_emb = self.pos_encoding(tgt_emb)
        tgt_emb = self.dropout(tgt_emb)
        
        # 通过解码器层
        decoder_output = tgt_emb
        for layer in self.decoder_layers:
            decoder_output = layer(decoder_output, encoder_output, 
                                 tgt_mask, src_mask)
        
        return decoder_output
    
    def forward(self, src, tgt, src_mask=None, tgt_mask=None):
        """完整前向传播"""
        # 编码
        encoder_output = self.encode(src, src_mask)
        
        # 解码
        decoder_output = self.decode(tgt, encoder_output, tgt_mask, src_mask)
        
        # 输出投影
        output = self.output_projection(decoder_output)
        
        return output

class TransformerDecoderLayer(nn.Module):
    """Transformer解码器层"""
    
    def __init__(self, d_model, n_heads, d_ff, dropout=0.1):
        super(TransformerDecoderLayer, self).__init__()
        
        # 掩码多头自注意力
        self.self_attention = MultiHeadAttention(d_model, n_heads, dropout)
        
        # 编码器-解码器注意力
        self.cross_attention = MultiHeadAttention(d_model, n_heads, dropout)
        
        # 前馈网络
        self.feed_forward = PositionwiseFeedForward(d_model, d_ff, dropout)
        
        # 层归一化
        self.norm1 = nn.LayerNorm(d_model)
        self.norm2 = nn.LayerNorm(d_model)
        self.norm3 = nn.LayerNorm(d_model)
        
        # Dropout
        self.dropout = nn.Dropout(dropout)
    
    def forward(self, x, encoder_output, tgt_mask=None, src_mask=None):
        """
        解码器层前向传播
        
        Args:
            x: 解码器输入
            encoder_output: 编码器输出
            tgt_mask: 目标序列掩码
            src_mask: 源序列掩码
        """
        # 第一个子层：掩码多头自注意力
        attn1 = self.self_attention(
            self.norm1(x), self.norm1(x), self.norm1(x), tgt_mask
        )
        x = x + self.dropout(attn1)
        
        # 第二个子层：编码器-解码器注意力
        attn2 = self.cross_attention(
            self.norm2(x), encoder_output, encoder_output, src_mask
        )
        x = x + self.dropout(attn2)
        
        # 第三个子层：前馈网络
        ff = self.feed_forward(self.norm3(x))
        x = x + self.dropout(ff)
        
        return x

# 使用示例
def create_transformer_model():
    """创建Transformer模型示例"""
    
    # 模型参数
    src_vocab_size = 10000  # 源语言词汇表大小
    tgt_vocab_size = 8000   # 目标语言词汇表大小
    d_model = 512           # 模型维度
    n_heads = 8             # 注意力头数
    n_layers = 6            # 层数
    d_ff = 2048             # 前馈网络维度
    
    # 创建模型
    model = Transformer(
        src_vocab_size=src_vocab_size,
        tgt_vocab_size=tgt_vocab_size,
        d_model=d_model,
        n_heads=n_heads,
        n_layers=n_layers,
        d_ff=d_ff
    )
    
    # 打印模型信息
    total_params = sum(p.numel() for p in model.parameters())
    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
    
    print(f"模型总参数量: {total_params:,}")
    print(f"可训练参数量: {trainable_params:,}")
    print(f"模型大小: {total_params * 4 / 1024 / 1024:.2f} MB")
    
    return model

# 测试模型
if __name__ == "__main__":
    model = create_transformer_model()
    
    # 创建测试数据
    batch_size = 2
    src_len = 20
    tgt_len = 15
    
    src = torch.randint(0, 10000, (batch_size, src_len))
    tgt = torch.randint(0, 8000, (batch_size, tgt_len))
    
    # 前向传播
    output = model(src, tgt)
    print(f"输出形状: {output.shape}")  # (batch_size, tgt_len, tgt_vocab_size)

class LayerNorm(nn.Module):
    """层归一化 - 数据的调味师"""
    
    def __init__(self, d_model, eps=1e-6):
        super(LayerNorm, self).__init__()
        
        # 可学习的缩放参数（就像调味料的用量）
        self.gamma = nn.Parameter(torch.ones(d_model))
        # 可学习的偏移参数（就像基础调味）
        self.beta = nn.Parameter(torch.zeros(d_model))
        # 防止除零的小数
        self.eps = eps
        
    def forward(self, x):
        """
        归一化过程：
        1. 计算平均值（这道菜的平均咸度）
        2. 计算标准差（咸度的变化范围）
        3. 标准化（调整到标准咸度）
        4. 缩放和偏移（根据个人口味微调）
        """
        # 计算均值和方差（在最后一个维度上）
        mean = x.mean(dim=-1, keepdim=True)
        var = x.var(dim=-1, keepdim=True, unbiased=False)
        
        # 归一化：(x - 平均值) / 标准差
        normalized = (x - mean) / torch.sqrt(var + self.eps)
        
        # 缩放和偏移：让模型学会最佳的"调味"
        return self.gamma * normalized + self.beta

class SparseAttention(nn.Module):
    """稀疏注意力实现"""
    
    def __init__(self, d_model, n_heads, sparsity_pattern='local'):
        super(SparseAttention, self).__init__()
        self.d_model = d_model
        self.n_heads = n_heads
        self.d_k = d_model // n_heads
        self.sparsity_pattern = sparsity_pattern
        
        self.W_q = nn.Linear(d_model, d_model)
        self.W_k = nn.Linear(d_model, d_model)
        self.W_v = nn.Linear(d_model, d_model)
        self.W_o = nn.Linear(d_model, d_model)
    
    def create_sparse_mask(self, seq_len):
        """创建稀疏注意力掩码"""
        mask = torch.zeros(seq_len, seq_len)
        
        if self.sparsity_pattern == 'local':
            # 局部注意力：只关注邻近位置
            window_size = 64
            for i in range(seq_len):
                start = max(0, i - window_size // 2)
                end = min(seq_len, i + window_size // 2 + 1)
                mask[i, start:end] = 1
                
        elif self.sparsity_pattern == 'strided':
            # 步长注意力：按固定步长采样
            stride = 8
            for i in range(seq_len):
                mask[i, ::stride] = 1
                mask[i, i] = 1  # 总是关注自己
                
        elif self.sparsity_pattern == 'random':
            # 随机注意力：随机选择位置
            sparsity_ratio = 0.1
            num_connections = int(seq_len * sparsity_ratio)
            for i in range(seq_len):
                indices = torch.randperm(seq_len)[:num_connections]
                mask[i, indices] = 1
                mask[i, i] = 1  # 总是关注自己
        
        return mask
    
    def forward(self, x):
        batch_size, seq_len, d_model = x.size()
        
        # 创建稀疏掩码
        sparse_mask = self.create_sparse_mask(seq_len).to(x.device)
        
        # 计算Q, K, V
        Q = self.W_q(x).view(batch_size, seq_len, self.n_heads, self.d_k).transpose(1, 2)
        K = self.W_k(x).view(batch_size, seq_len, self.n_heads, self.d_k).transpose(1, 2)
        V = self.W_v(x).view(batch_size, seq_len, self.n_heads, self.d_k).transpose(1, 2)
        
        # 计算注意力分数
        scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.d_k)
        
        # 应用稀疏掩码
        scores = scores.masked_fill(sparse_mask.unsqueeze(0).unsqueeze(0) == 0, -1e9)
        
        # Softmax和加权求和
        attention_weights = F.softmax(scores, dim=-1)
        output = torch.matmul(attention_weights, V)
        
        # 重塑和输出投影
        output = output.transpose(1, 2).contiguous().view(batch_size, seq_len, d_model)
        return self.W_o(output)

class LinearAttention(nn.Module):
    """线性注意力实现"""
    
    def __init__(self, d_model, n_heads):
        super(LinearAttention, self).__init__()
        self.d_model = d_model
        self.n_heads = n_heads
        self.d_k = d_model // n_heads
        
        self.W_q = nn.Linear(d_model, d_model)
        self.W_k = nn.Linear(d_model, d_model)
        self.W_v = nn.Linear(d_model, d_model)
        self.W_o = nn.Linear(d_model, d_model)
    
    def kernel_function(self, x):
        """核函数：用ELU+1近似softmax"""
        return F.elu(x) + 1
    
    def forward(self, x):
        batch_size, seq_len, d_model = x.size()
        
        # 计算Q, K, V
        Q = self.W_q(x).view(batch_size, seq_len, self.n_heads, self.d_k).transpose(1, 2)
        K = self.W_k(x).view(batch_size, seq_len, self.n_heads, self.d_k).transpose(1, 2)
        V = self.W_v(x).view(batch_size, seq_len, self.n_heads, self.d_k).transpose(1, 2)
        
        # 应用核函数
        Q = self.kernel_function(Q)
        K = self.kernel_function(K)
        
        # 线性注意力计算：O(nd²)而不是O(n²d)
        # 计算K^T V
        KV = torch.matmul(K.transpose(-2, -1), V)  # (batch, heads, d_k, d_k)
        
        # 计算Q(K^T V)
        output = torch.matmul(Q, KV)  # (batch, heads, seq_len, d_k)
        
        # 归一化
        normalizer = torch.matmul(Q, K.sum(dim=-2, keepdim=True).transpose(-2, -1))
        output = output / (normalizer + 1e-6)
        
        # 重塑和输出投影
        output = output.transpose(1, 2).contiguous().view(batch_size, seq_len, d_model)
        return self.W_o(output)

def flash_attention_concept(Q, K, V, block_size=64):
    """
    Flash Attention概念实现（简化版）
    实际实现需要CUDA kernel优化
    """
    batch_size, n_heads, seq_len, d_k = Q.shape
    
    # 分块处理
    num_blocks = (seq_len + block_size - 1) // block_size
    output = torch.zeros_like(Q)
    
    for i in range(num_blocks):
        # 当前查询块
        start_i = i * block_size
        end_i = min((i + 1) * block_size, seq_len)
        Q_block = Q[:, :, start_i:end_i, :]
        
        # 初始化输出和归一化项
        O_block = torch.zeros_like(Q_block)
        l_block = torch.zeros(batch_size, n_heads, end_i - start_i, 1)
        m_block = torch.full((batch_size, n_heads, end_i - start_i, 1), -float('inf'))
        
        for j in range(num_blocks):
            # 当前键值块
            start_j = j * block_size
            end_j = min((j + 1) * block_size, seq_len)
            K_block = K[:, :, start_j:end_j, :]
            V_block = V[:, :, start_j:end_j, :]
            
            # 计算注意力分数
            S_block = torch.matmul(Q_block, K_block.transpose(-2, -1)) / math.sqrt(d_k)
            
            # 在线softmax更新
            m_new = torch.maximum(m_block, S_block.max(dim=-1, keepdim=True)[0])
            alpha = torch.exp(m_block - m_new)
            beta = torch.exp(S_block - m_new)
            
            l_new = alpha * l_block + beta.sum(dim=-1, keepdim=True)
            
            # 更新输出
            O_block = (alpha * l_block * O_block + torch.matmul(beta, V_block)) / l_new
            
            # 更新状态
            l_block = l_new
            m_block = m_new
        
        output[:, :, start_i:end_i, :] = O_block
    
    return output

class CheckpointedTransformerLayer(nn.Module):
    """使用梯度检查点的Transformer层"""
    
    def __init__(self, d_model, n_heads, d_ff, dropout=0.1):
        super(CheckpointedTransformerLayer, self).__init__()
        self.attention = MultiHeadAttention(d_model, n_heads, dropout)
        self.feed_forward = PositionwiseFeedForward(d_model, d_ff, dropout)
        self.norm1 = nn.LayerNorm(d_model)
        self.norm2 = nn.LayerNorm(d_model)
        self.dropout = nn.Dropout(dropout)
    
    def forward(self, x, mask=None):
        # 使用梯度检查点
        x = checkpoint(self._attention_block, x, mask)
        x = checkpoint(self._feedforward_block, x)
        return x
    
    def _attention_block(self, x, mask):
        """注意力子块"""
        attn_output = self.attention(self.norm1(x), self.norm1(x), self.norm1(x), mask)
        return x + self.dropout(attn_output)
    
    def _feedforward_block(self, x):
        """前馈网络子块"""
        ff_output = self.feed_forward(self.norm2(x))
        return x + self.dropout(ff_output)

# 使用torch.utils.checkpoint
from torch.utils.checkpoint import checkpoint

def checkpointed_forward(layer, x, mask=None):
    """使用检查点的前向传播"""
    if x.requires_grad:
        return checkpoint(layer, x, mask)
    else:
        return layer(x, mask)

class MixedPrecisionTransformer(nn.Module):
    """混合精度Transformer"""
    
    def __init__(self, *args, **kwargs):
        super(MixedPrecisionTransformer, self).__init__()
        # ... 初始化层 ...
        
        # 自动混合精度
        self.scaler = torch.cuda.amp.GradScaler()
    
    def forward(self, x):
        # 使用autocast进行混合精度计算
        with torch.cuda.amp.autocast():
            # ... 前向传播 ...
            return output
    
    def training_step(self, batch):
        """训练步骤"""
        self.optimizer.zero_grad()
        
        # 混合精度前向传播
        with torch.cuda.amp.autocast():
            output = self(batch['input'])
            loss = self.criterion(output, batch['target'])
        
        # 缩放损失并反向传播
        self.scaler.scale(loss).backward()
        
        # 更新参数
        self.scaler.step(self.optimizer)
        self.scaler.update()
        
        return loss

class CompressedTransformer(nn.Module):
    """压缩的Transformer模型"""
    
    def __init__(self, d_model, n_heads, n_layers, compression_config):
        super(CompressedTransformer, self).__init__()
        
        self.compression_config = compression_config
        
        # 参数共享：多层共享权重
        if compression_config.get('weight_sharing', False):
            shared_layer = TransformerEncoderLayer(d_model, n_heads, d_model * 4)
            self.layers = nn.ModuleList([shared_layer] * n_layers)
        else:
            self.layers = nn.ModuleList([
                TransformerEncoderLayer(d_model, n_heads, d_model * 4)
                for _ in range(n_layers)
            ])
        
        # 量化配置
        if compression_config.get('quantization', False):
            self.apply(self._quantize_weights)
    
    def _quantize_weights(self, module):
        """权重量化"""
        if isinstance(module, nn.Linear):
            # 8位量化
            weight = module.weight.data
            scale = weight.abs().max() / 127
            quantized_weight = torch.round(weight / scale).clamp(-128, 127)
            module.weight.data = quantized_weight * scale
    
    def forward(self, x):
        for layer in self.layers:
            x = layer(x)
        return x

def clip_gradients(model, max_norm=1.0):
    """梯度裁剪"""
    torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm)

# 在训练循环中使用
loss.backward()
clip_gradients(model, max_norm=1.0)
optimizer.step()

class WarmupLRScheduler:
    """带预热的学习率调度器"""
    
    def __init__(self, optimizer, d_model, warmup_steps=4000):
        self.optimizer = optimizer
        self.d_model = d_model
        self.warmup_steps = warmup_steps
        self.step_num = 0
    
    def step(self):
        self.step_num += 1
        lr = self._get_lr()
        for param_group in self.optimizer.param_groups:
            param_group['lr'] = lr
    
    def _get_lr(self):
        """计算当前学习率"""
        return (self.d_model ** -0.5) * min(
            self.step_num ** -0.5,
            self.step_num * (self.warmup_steps ** -1.5)
        )

def init_transformer_weights(module):
    """Transformer权重初始化"""
    if isinstance(module, nn.Linear):
        # Xavier均匀初始化
        nn.init.xavier_uniform_(module.weight)
        if module.bias is not None:
            nn.init.constant_(module.bias, 0)
    elif isinstance(module, nn.Embedding):
        # 正态分布初始化
        nn.init.normal_(module.weight, mean=0, std=0.02)
    elif isinstance(module, nn.LayerNorm):
        # 层归一化参数初始化
        nn.init.constant_(module.bias, 0)
        nn.init.constant_(module.weight, 1.0)

# 应用初始化
model.apply(init_transformer_weights)

class StableTransformerTrainer:
    """稳定的Transformer训练器"""
    
    def __init__(self, model, optimizer, scheduler=None):
        self.model = model
        self.optimizer = optimizer
        self.scheduler = scheduler
        
        # 初始化权重
        self.model.apply(init_transformer_weights)
        
        # 梯度裁剪参数
        self.max_grad_norm = 1.0
        
        # 损失缩放（用于混合精度）
        self.scaler = torch.cuda.amp.GradScaler()
    
    def train_step(self, batch):
        """稳定的训练步骤"""
        self.model.train()
        self.optimizer.zero_grad()
        
        # 混合精度前向传播
        with torch.cuda.amp.autocast():
            outputs = self.model(batch['input_ids'], batch['attention_mask'])
            loss = F.cross_entropy(outputs.logits.view(-1, outputs.logits.size(-1)), 
                                 batch['labels'].view(-1))
        
        # 缩放损失并反向传播
        self.scaler.scale(loss).backward()
        
        # 梯度裁剪
        self.scaler.unscale_(self.optimizer)
        torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.max_grad_norm)
        
        # 更新参数
        self.scaler.step(self.optimizer)
        self.scaler.update()
        
        # 学习率调度
        if self.scheduler:
            self.scheduler.step()
        
        return loss.item()
    
    def validate_gradients(self):
        """验证梯度健康状态"""
        total_norm = 0
        param_count = 0
        
        for name, param in self.model.named_parameters():
            if param.grad is not None:
                param_norm = param.grad.data.norm(2)
                total_norm += param_norm.item() ** 2
                param_count += 1
                
                # 检查异常梯度
                if torch.isnan(param.grad).any():
                    print(f"警告: {name} 包含NaN梯度")
                if torch.isinf(param.grad).any():
                    print(f"警告: {name} 包含无穷梯度")
        
        total_norm = total_norm ** (1. / 2)
        print(f"总梯度范数: {total_norm:.4f}, 参数数量: {param_count}")
        
        return total_norm

# 使用示例
def create_stable_training_setup():
    """创建稳定的训练配置"""
    
    # 模型配置
    model = Transformer(
        src_vocab_size=10000,
        tgt_vocab_size=10000,
        d_model=512,
        n_heads=8,
        n_layers=6
    )
    
    # 优化器配置
    optimizer = torch.optim.AdamW(
        model.parameters(),
        lr=1e-4,
        betas=(0.9, 0.98),
        eps=1e-9,
        weight_decay=0.01
    )
    
    # 学习率调度器
    scheduler = WarmupLRScheduler(optimizer, d_model=512, warmup_steps=4000)
    
    # 创建训练器
    trainer = StableTransformerTrainer(model, optimizer, scheduler)
    
    return trainer

# 训练循环示例
def stable_training_loop(trainer, dataloader, num_epochs=10):
    """稳定的训练循环"""
    
    for epoch in range(num_epochs):
        total_loss = 0
        num_batches = 0
        
        for batch_idx, batch in enumerate(dataloader):
            # 训练步骤
            loss = trainer.train_step(batch)
            total_loss += loss
            num_batches += 1
            
            # 定期检查梯度
            if batch_idx % 100 == 0:
                grad_norm = trainer.validate_gradients()
                print(f"Epoch {epoch}, Batch {batch_idx}, Loss: {loss:.4f}, Grad Norm: {grad_norm:.4f}")
        
        avg_loss = total_loss / num_batches
        print(f"Epoch {epoch} 完成, 平均损失: {avg_loss:.4f}")

# 自动混合精度训练
scaler = GradScaler()
optimizer = torch.optim.AdamW(model.parameters())

for batch in dataloader:
    optimizer.zero_grad()
    
    # 前向传播使用自动混合精度
    with autocast():
        outputs = model(batch['input_ids'])
        loss = criterion(outputs, batch['labels'])
    
    # 反向传播
    scaler.scale(loss).backward()
    scaler.step(optimizer)
    scaler.update()

class DataQualityMetrics:
    """数据质量评估工具"""
    
    def __init__(self):
        self.language_detector = LanguageDetector()
        self.profanity_filter = ProfanityFilter()
        
    def assess_quality(self, text):
        """评估单个文本质量"""
        metrics = {}
        
        # 1. 语言检测
        metrics['language'] = self.language_detector.detect(text)
        metrics['language_confidence'] = self.language_detector.confidence()
        
        # 2. 长度统计
        metrics['char_count'] = len(text)
        metrics['word_count'] = len(text.split())
        metrics['avg_word_length'] = np.mean([len(word) for word in text.split()])
        
        # 3. 重复性检测
        lines = text.split('\n')
        metrics['duplicate_lines'] = len(lines) - len(set(lines))
        
        # 4. 特殊字符比例
        special_chars = sum(1 for c in text if not c.isalnum() and not c.isspace())
        metrics['special_char_ratio'] = special_chars / len(text)
        
        # 5. 有害内容检测
        metrics['has_profanity'] = self.profanity_filter.contains_profanity(text)
        
        # 6. 质量评分
        metrics['quality_score'] = self.calculate_quality_score(metrics)
        
        return metrics
    
    def calculate_quality_score(self, metrics):
        """计算综合质量评分"""
        score = 1.0
        
        # 语言置信度惩罚
        if metrics['language_confidence'] < 0.8:
            score *= 0.8
        
        # 长度惩罚
        if metrics['word_count'] < 10:
            score *= 0.5
        elif metrics['word_count'] > 10000:
            score *= 0.9
        
        # 重复性惩罚
        if metrics['duplicate_lines'] > 0:
            score *= (1 - metrics['duplicate_lines'] / 100)
        
        # 特殊字符惩罚
        if metrics['special_char_ratio'] > 0.3:
            score *= 0.7
        
        # 有害内容惩罚
        if metrics['has_profanity']:
            score = 0.0
        
        return max(0.0, min(1.0, score))

class CausalLanguageModel(nn.Module):
    """因果语言模型"""
    
    def __init__(self, vocab_size, d_model, n_layers, n_heads):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, d_model)
        self.pos_encoding = PositionalEncoding(d_model)
        self.transformer_blocks = nn.ModuleList([
            TransformerBlock(d_model, n_heads) for _ in range(n_layers)
        ])
        self.ln_f = nn.LayerNorm(d_model)
        self.lm_head = nn.Linear(d_model, vocab_size)
        
    def forward(self, input_ids, labels=None):
        # 词嵌入 + 位置编码
        x = self.embedding(input_ids)
        x = self.pos_encoding(x)
        
        # Transformer层
        for block in self.transformer_blocks:
            x = block(x, causal_mask=True)
        
        # 最终归一化
        x = self.ln_f(x)
        
        # 语言模型头
        logits = self.lm_head(x)
        
        # 计算损失
        if labels is not None:
            # 下一个token预测任务
            shift_logits = logits[..., :-1, :].contiguous()
            shift_labels = labels[..., 1:].contiguous()
            
            loss_fct = nn.CrossEntropyLoss()
            loss = loss_fct(
                shift_logits.view(-1, shift_logits.size(-1)),
                shift_labels.view(-1)
            )
            return loss, logits
        
        return logits

class MaskedLanguageModel(nn.Module):
    """掩码语言模型"""
    
    def forward(self, input_ids, labels=None):
        # 获取编码器输出
        encoder_output = self.encoder(input_ids)
        
        # MLM预测头
        mlm_logits = self.mlm_head(encoder_output)
        
        if labels is not None:
            # 只计算被掩码位置的损失
            loss_fct = nn.CrossEntropyLoss()
            
            # labels中-100表示不计算损失的位置
            active_loss = labels.view(-1) != -100
            active_logits = mlm_logits.view(-1, mlm_logits.size(-1))[active_loss]
            active_labels = labels.view(-1)[active_loss]
            
            loss = loss_fct(active_logits, active_labels)
            return loss, mlm_logits
        
        return mlm_logits

def create_mlm_data(texts, tokenizer, mask_prob=0.15):
    """创建MLM训练数据"""
    inputs = tokenizer(texts, return_tensors='pt', padding=True, truncation=True)
    
    input_ids = inputs['input_ids'].clone()
    labels = input_ids.clone()
    
    # 创建随机掩码
    probability_matrix = torch.full(labels.shape, mask_prob)
    special_tokens_mask = tokenizer.get_special_tokens_mask(
        labels.tolist(), already_has_special_tokens=True
    )
    probability_matrix.masked_fill_(torch.tensor(special_tokens_mask, dtype=torch.bool), 0.0)
    
    masked_indices = torch.bernoulli(probability_matrix).bool()
    labels[~masked_indices] = -100  # 不计算损失
    
    # 80%替换为[MASK], 10%替换为随机token, 10%保持不变
    indices_replaced = torch.bernoulli(torch.full(labels.shape, 0.8)).bool() & masked_indices
    input_ids[indices_replaced] = tokenizer.mask_token_id
    
    indices_random = torch.bernoulli(torch.full(labels.shape, 0.5)).bool() & masked_indices & ~indices_replaced
    random_words = torch.randint(len(tokenizer), labels.shape, dtype=torch.long)
    input_ids[indices_random] = random_words[indices_random]
    
    return {'input_ids': input_ids, 'labels': labels}

# DeepSpeed ZeRO配置示例
deepspeed_config = {
    "train_batch_size": 32,
    "gradient_accumulation_steps": 4,
    "optimizer": {
        "type": "AdamW",
        "params": {
            "lr": 1e-4,
            "betas": [0.9, 0.999],
            "eps": 1e-8,
            "weight_decay": 0.01
        }
    },
    "scheduler": {
        "type": "WarmupLR",
        "params": {
            "warmup_min_lr": 0,
            "warmup_max_lr": 1e-4,
            "warmup_num_steps": 1000
        }
    },
    "zero_optimization": {
        "stage": 3,  # ZeRO Stage 3: 参数、梯度、优化器状态都分区
        "offload_optimizer": {
            "device": "cpu",  # 优化器状态卸载到CPU
            "pin_memory": True
        },
        "offload_param": {
            "device": "cpu",  # 参数卸载到CPU
            "pin_memory": True
        },
        "overlap_comm": True,  # 通信与计算重叠
        "contiguous_gradients": True,
        "sub_group_size": 1e9,
        "reduce_bucket_size": 1e6,
        "stage3_prefetch_bucket_size": 1e6,
        "stage3_param_persistence_threshold": 1e4
    },
    "activation_checkpointing": {
        "partition_activations": True,
        "cpu_checkpointing": True,
        "contiguous_memory_optimization": False,
        "number_checkpoints": None,
        "synchronize_checkpoint_boundary": False
    },
    "wall_clock_breakdown": False
}

# 使用DeepSpeed训练
import deepspeed

model_engine, optimizer, _, _ = deepspeed.initialize(
    args=args,
    model=model,
    model_parameters=model.parameters(),
    config=deepspeed_config
)

for batch in dataloader:
    loss = model_engine(batch)
    model_engine.backward(loss)
    model_engine.step()

class FineTuningTrainer:
    """全参数微调训练器"""
    
    def __init__(self, model, tokenizer, config):
        self.model = model
        self.tokenizer = tokenizer
        self.config = config
        
        # 设置优化器
        self.optimizer = torch.optim.AdamW(
            model.parameters(),
            lr=config.learning_rate,
            weight_decay=config.weight_decay
        )
        
        # 学习率调度器
        self.scheduler = self.get_scheduler()
        
    def fine_tune(self, train_dataset, eval_dataset):
        """执行微调训练"""
        
        train_dataloader = DataLoader(
            train_dataset, 
            batch_size=self.config.batch_size,
            shuffle=True
        )
        
        self.model.train()
        global_step = 0
        
        for epoch in range(self.config.num_epochs):
            epoch_loss = 0
            
            for batch in train_dataloader:
                # 前向传播
                outputs = self.model(**batch)
                loss = outputs.loss
                
                # 反向传播
                loss.backward()
                
                # 梯度裁剪
                torch.nn.utils.clip_grad_norm_(
                    self.model.parameters(), 
                    self.config.max_grad_norm
                )
                
                # 优化器步骤
                self.optimizer.step()
                self.scheduler.step()
                self.optimizer.zero_grad()
                
                epoch_loss += loss.item()
                global_step += 1
                
                # 定期评估
                if global_step % self.config.eval_steps == 0:
                    self.evaluate(eval_dataset)
            
            print(f"Epoch {epoch}, Loss: {epoch_loss / len(train_dataloader)}")
    
    def evaluate(self, eval_dataset):
        """评估模型性能"""
        self.model.eval()
        total_loss = 0
        
        eval_dataloader = DataLoader(eval_dataset, batch_size=self.config.batch_size)
        
        with torch.no_grad():
            for batch in eval_dataloader:
                outputs = self.model(**batch)
                total_loss += outputs.loss.item()
        
        avg_loss = total_loss / len(eval_dataloader)
        print(f"Eval Loss: {avg_loss}")
        
        self.model.train()
        return avg_loss

class LoRALayer(nn.Module):
    """LoRA适配层"""
    
    def __init__(self, in_features, out_features, rank=16, alpha=32, dropout=0.1):
        super().__init__()
        self.rank = rank
        self.alpha = alpha
        
        # 原始线性层（冻结）
        self.linear = nn.Linear(in_features, out_features, bias=False)
        self.linear.weight.requires_grad = False
        
        # LoRA分解矩阵
        self.lora_A = nn.Parameter(torch.randn(rank, in_features) * 0.01)
        self.lora_B = nn.Parameter(torch.zeros(out_features, rank))
        
        self.dropout = nn.Dropout(dropout)
        self.scaling = alpha / rank
        
    def forward(self, x):
        # 原始输出
        original_output = self.linear(x)
        
        # LoRA输出
        lora_output = self.dropout(x) @ self.lora_A.T @ self.lora_B.T
        
        return original_output + lora_output * self.scaling

class LoRAModel(nn.Module):
    """应用LoRA的模型包装器"""
    
    def __init__(self, base_model, target_modules=['q_proj', 'v_proj'], rank=16):
        super().__init__()
        self.base_model = base_model
        self.lora_layers = nn.ModuleDict()
        
        # 为目标模块添加LoRA层
        for name, module in base_model.named_modules():
            if any(target in name for target in target_modules):
                if isinstance(module, nn.Linear):
                    # 替换为LoRA层
                    lora_layer = LoRALayer(
                        module.in_features,
                        module.out_features,
                        rank=rank
                    )
                    # 复制原始权重
                    lora_layer.linear.weight.data.copy_(module.weight.data)
                    
                    # 替换模块
                    parent_name = '.'.join(name.split('.')[:-1])
                    child_name = name.split('.')[-1]
                    parent_module = self.get_submodule(parent_name)
                    setattr(parent_module, child_name, lora_layer)
    
    def forward(self, *args, **kwargs):
        return self.base_model(*args, **kwargs)
    
    def get_lora_parameters(self):
        """获取LoRA参数用于训练"""
        lora_params = []
        for name, param in self.named_parameters():
            if 'lora_' in name:
                lora_params.append(param)
        return lora_params

class AdapterLayer(nn.Module):
    """Adapter调优层"""
    
    def __init__(self, d_model, bottleneck_size=64):
        super().__init__()
        self.down_project = nn.Linear(d_model, bottleneck_size)
        self.up_project = nn.Linear(bottleneck_size, d_model)
        self.activation = nn.ReLU()
        self.dropout = nn.Dropout(0.1)
        
        # 残差连接的门控机制
        self.gate = nn.Parameter(torch.zeros(1))
        
    def forward(self, x):
        # 下投影 -> 激活 -> 上投影
        adapter_output = self.up_project(
            self.activation(self.down_project(x))
        )
        adapter_output = self.dropout(adapter_output)
        
        # 门控残差连接
        return x + self.gate * adapter_output

class AdapterTransformerBlock(nn.Module):
    """带Adapter的Transformer块"""
    
    def __init__(self, transformer_block, adapter_size=64):
        super().__init__()
        self.transformer_block = transformer_block
        
        # 冻结原始参数
        for param in transformer_block.parameters():
            param.requires_grad = False
        
        # 添加Adapter层
        d_model = transformer_block.self_attention.d_model
        self.adapter1 = AdapterLayer(d_model, adapter_size)
        self.adapter2 = AdapterLayer(d_model, adapter_size)
    
    def forward(self, x, mask=None):
        # 自注意力 + Adapter
        attn_output = self.transformer_block.self_attention(x, mask)
        x = self.transformer_block.norm1(x + attn_output)
        x = self.adapter1(x)  # 第一个Adapter
        
        # 前馈网络 + Adapter
        ff_output = self.transformer_block.feed_forward(x)
        x = self.transformer_block.norm2(x + ff_output)
        x = self.adapter2(x)  # 第二个Adapter
        
        return x