fix bugs

examples/aishell/asr1/conf/chunk_roformer.yaml

@@ -18,7 +18,7 @@ encoder_conf:
  cnn_module_kernel: 15
  use_cnn_module: True
  activation_type: 'swish'
- pos_enc_layer_type: 'rpoe_pos' # abs_pos, rel_pos, rope_pos
+ pos_enc_layer_type: 'rope_pos' # abs_pos, rel_pos, rope_pos
  selfattention_layer_type: 'rel_selfattn' # unused
  causal: true
  use_dynamic_chunk: true
@@ -30,7 +30,7 @@ decoder_conf:
  attention_heads: 4
  linear_units: 2048
  num_blocks: 6
- r_num_blocks: 3 # only for bitransformer
+ r_num_blocks: 0 # only for bitransformer
  dropout_rate: 0.1 # sublayer output dropout
  positional_dropout_rate: 0.1
  self_attention_dropout_rate: 0.0
@@ -39,7 +39,7 @@ decoder_conf:
 model_conf:
  ctc_weight: 0.3
  lsm_weight: 0.1 # label smoothing option
- reverse_weight: 0.3 # only for bitransformer
+ reverse_weight: 0.0 # only for bitransformer
  length_normalized_loss: false
  init_type: 'kaiming_uniform' # !Warning: need to convergence
 

examples/aishell/asr1/conf/chunk_roformer_bidecoder.yaml

@@ -18,7 +18,7 @@ encoder_conf:
  cnn_module_kernel: 15
  use_cnn_module: True
  activation_type: 'swish'
- pos_enc_layer_type: 'rpoe_pos' # abs_pos, rel_pos, rope_pos
+ pos_enc_layer_type: 'rope_pos' # abs_pos, rel_pos, rope_pos
  selfattention_layer_type: 'rel_selfattn' # unused
  causal: true
  use_dynamic_chunk: true

paddlespeech/s2t/models/u2/u2.py

@@ -145,7 +145,6 @@ def forward(
  text_lengths)
  ctc_time = time.time() - start
  #logger.debug(f"ctc time: {ctc_time}")
-
  if loss_ctc is None:
  loss = loss_att
  elif loss_att is None:
@@ -916,6 +915,8 @@ def _init_from_config(cls, configs: dict):
  decoder_type = configs.get('decoder', 'transformer')
  logger.debug(f"U2 Decoder type: {decoder_type}")
  if decoder_type == 'transformer':
+ configs['model_conf'].pop('reverse_weight', None)
+ configs['decoder_conf'].pop('r_num_blocks', None)
  decoder = TransformerDecoder(vocab_size,
  encoder.output_size(),
  **configs['decoder_conf'])

paddlespeech/s2t/modules/attention.py

@@ -16,6 +16,7 @@
 """Multi-Head Attention layer definition."""
 import math
 from typing import Tuple
+from typing import List
 
 import paddle
 from paddle import nn
@@ -418,25 +419,27 @@ def align(self, tensor: paddle.Tensor, axes: List[int], ndim=None):
  def apply_rotary_position_embeddings(self, sinusoidal, *tensors):
  """应用RoPE到tensors中
  其中，sinusoidal.shape=[B, T, D]，tensors为tensor的列表，而
- tensor.shape=[B, T, ..., D], or (B,T,H,D/H)
+ tensor.shape=[B, T, ..., D], or (B,H,T,D/H)
  """
  assert len(tensors) > 0, 'at least one input tensor'
  assert all(
  [tensor.shape == tensors[0].shape
  for tensor in tensors[1:]]), 'all tensors must have the same shape'
 
+ # (B,H,T,D)
  ndim = tensors[0].dim()
+ _,H,T,D = tensors[0].shape
 
  # sinusoidal shape same with tensors[0]
- # [B,T,D] -> [B,T,1,D]
- sinusoidal = self.align(sinusoidal, [0, 1, -1], ndim)
+ # [B,T,D] -> [B,T,H,D/H] -> (B,H,T,D/H)
+ # sinusoidal = self.align(sinusoidal, [0, 1, -1], ndim)
+ sinusoidal = sinusoidal.reshape((1, T, H, D)).transpose([0, 2, 1, 3])
 
  # http:https://man.hubwiz.com/docset/TensorFlow.docset/Contents/Resources/Documents/api_docs/python/tf/keras/backend/repeat_elements.html
  # like np.repeat, x (s1, s2, s3), axis 1, (s1, s2*rep, s3)
  # [b,T, ..., d/2] -> [b,T, ..., d]
  cos_pos = paddle.repeat_interleave(sinusoidal[..., 1::2], 2, axis=-1)
  sin_pos = paddle.repeat_interleave(sinusoidal[..., 0::2], 2, axis=-1)
-
  outputs = []
  for tensor in tensors:
  # x2 = [-x2, x1, -x4, x3, ..., -x_d, x_{d-1}]
@@ -501,7 +504,7 @@ def forward(self,
  new_cache = paddle.concat((k, v), axis=-1)
 
  # f{q,k}(x_m, m) = R^d_{\theta, m} W_{q,k} x_m, m is position index
- q, k = self.apply_rotary_position_embeddings(pos_emb, [q, k])
+ q, k = self.apply_rotary_position_embeddings(pos_emb, q, k)
  # dot(q, k)
  scores = paddle.matmul(q, k, transpose_y=True) / math.sqrt(self.d_k)
  return self.forward_attention(v, scores, mask), new_cache

paddlespeech/s2t/modules/encoder.py

@@ -477,9 +477,10 @@ def __init__(self,
  activation = get_activation(activation_type)
 
  # self-attention module definition
+ encoder_dim = output_size
  if pos_enc_layer_type == "abs_pos":
  encoder_selfattn_layer = MultiHeadedAttention
- encoder_selfattn_layer_args = (attention_heads, output_size,
+ encoder_selfattn_layer_args = (attention_heads, encoder_dim,
  attention_dropout_rate)
  elif pos_enc_layer_type == "rel_pos":
  encoder_selfattn_layer = RelPositionMultiHeadedAttention
@@ -495,16 +496,16 @@ def __init__(self,
 
  # feed-forward module definition
  positionwise_layer = PositionwiseFeedForward
- positionwise_layer_args = (output_size, linear_units, dropout_rate,
+ positionwise_layer_args = (encoder_dim, linear_units, dropout_rate,
  activation)
  # convolution module definition
  convolution_layer = ConvolutionModule
- convolution_layer_args = (output_size, cnn_module_kernel, activation,
+ convolution_layer_args = (encoder_dim, cnn_module_kernel, activation,
  cnn_module_norm, causal)
 
  self.encoders = nn.LayerList([
  ConformerEncoderLayer(
- size=output_size,
+ size=encoder_dim,
  self_attn=encoder_selfattn_layer(*encoder_selfattn_layer_args),
  feed_forward=positionwise_layer(*positionwise_layer_args),
  feed_forward_macaron=positionwise_layer(