add blank_penalty for offline transducer

python-api-examples/non_streaming_server.py

@@ -383,6 +383,19 @@ def add_hotwords_args(parser: argparse.ArgumentParser):
         """,
     )
 
+def add_blank_penalty_args(parser: argparse.ArgumentParser):
+    parser.add_argument(
+        "--blank-penalty",
+        type=float,
+        default=0.0,
+        help="""
+        The penalty applied on blank symbol during decoding.
+        Note: It is a positive value that would be applied to logits like
+        this `logits[:, 0] -= blank_penalty` (suppose logits.shape is
+        [batch_size, vocab] and blank id is 0).
+        """,
+    )
+
 
 def check_args(args):
     if not Path(args.tokens).is_file():
@@ -414,6 +427,7 @@ def get_args():
     add_feature_config_args(parser)
     add_decoding_args(parser)
     add_hotwords_args(parser)
+    add_blank_penalty_args(parser)
 
     parser.add_argument(
         "--port",
@@ -862,6 +876,7 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer:
             max_active_paths=args.max_active_paths,
             hotwords_file=args.hotwords_file,
             hotwords_score=args.hotwords_score,
+            blank_penalty=args.blank_penalty,
             provider=args.provider,
         )
     elif args.paraformer:

python-api-examples/offline-decode-files.py

@@ -231,6 +231,18 @@ def get_args():
         """,
     )
 
+    parser.add_argument(
+        "--blank-penalty",
+        type=float,
+        default=0.0,
+        help="""
+        The penalty applied on blank symbol during decoding.
+        Note: It is a positive value that would be applied to logits like
+        this `logits[:, 0] -= blank_penalty` (suppose logits.shape is
+        [batch_size, vocab] and blank id is 0).
+        """,
+    )
+
     parser.add_argument(
         "--decoding-method",
         type=str,
@@ -335,6 +347,7 @@ def main():
             decoding_method=args.decoding_method,
             hotwords_file=args.hotwords_file,
             hotwords_score=args.hotwords_score,
+            blank_penalty=args.blank_penalty,
             debug=args.debug,
         )
     elif args.paraformer:

python-api-examples/vad-with-non-streaming-asr.py

@@ -177,6 +177,18 @@ def get_args():
         """,
     )
 
+    parser.add_argument(
+        "--blank-penalty",
+        type=float,
+        default=0.0,
+        help="""
+        The penalty applied on blank symbol during decoding.
+        Note: It is a positive value that would be applied to logits like
+        this `logits[:, 0] -= blank_penalty` (suppose logits.shape is
+        [batch_size, vocab] and blank id is 0).
+        """,
+    )
+
     parser.add_argument(
         "--decoding-method",
         type=str,
@@ -237,6 +249,7 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer:
             sample_rate=args.sample_rate,
             feature_dim=args.feature_dim,
             decoding_method=args.decoding_method,
+            blank_penalty=args.blank_penalty,
             debug=args.debug,
         )
     elif args.paraformer:

sherpa-onnx/csrc/math.h

@@ -96,6 +96,14 @@ void LogSoftmax(T *in, int32_t w, int32_t h) {
   }
 }
 
+template <typename T>
+void SubtractBlank(T *in, int32_t w, int32_t h, int32_t blank_idx, float blank_penalty) {
+  for (int32_t i = 0; i != h; ++i) {
+    in[blank_idx] -= blank_penalty;
+    in += w;
+  }
+}
+
 template <class T>
 std::vector<int32_t> TopkIndex(const T *vec, int32_t size, int32_t topk) {
   std::vector<int32_t> vec_index(size);

sherpa-onnx/csrc/offline-recognizer-transducer-impl.h

@@ -79,15 +79,15 @@ class OfflineRecognizerTransducerImpl : public OfflineRecognizerImpl {
     }
     if (config_.decoding_method == "greedy_search") {
       decoder_ =
-          std::make_unique<OfflineTransducerGreedySearchDecoder>(model_.get());
+          std::make_unique<OfflineTransducerGreedySearchDecoder>(model_.get(), config_.blank_penalty);
     } else if (config_.decoding_method == "modified_beam_search") {
       if (!config_.lm_config.model.empty()) {
         lm_ = OfflineLM::Create(config.lm_config);
       }
 
       decoder_ = std::make_unique<OfflineTransducerModifiedBeamSearchDecoder>(
           model_.get(), lm_.get(), config_.max_active_paths,
-          config_.lm_config.scale);
+          config_.lm_config.scale, config_.blank_penalty);
     } else {
       SHERPA_ONNX_LOGE("Unsupported decoding method: %s",
                        config_.decoding_method.c_str());
@@ -104,15 +104,15 @@ class OfflineRecognizerTransducerImpl : public OfflineRecognizerImpl {
                                                         config_.model_config)) {
     if (config_.decoding_method == "greedy_search") {
       decoder_ =
-          std::make_unique<OfflineTransducerGreedySearchDecoder>(model_.get());
+          std::make_unique<OfflineTransducerGreedySearchDecoder>(model_.get(), config_.blank_penalty);
     } else if (config_.decoding_method == "modified_beam_search") {
       if (!config_.lm_config.model.empty()) {
         lm_ = OfflineLM::Create(mgr, config.lm_config);
       }
 
       decoder_ = std::make_unique<OfflineTransducerModifiedBeamSearchDecoder>(
           model_.get(), lm_.get(), config_.max_active_paths,
-          config_.lm_config.scale);
+          config_.lm_config.scale, config_.blank_penalty);
     } else {
       SHERPA_ONNX_LOGE("Unsupported decoding method: %s",
                        config_.decoding_method.c_str());

sherpa-onnx/csrc/offline-recognizer.cc

@@ -28,6 +28,12 @@ void OfflineRecognizerConfig::Register(ParseOptions *po) {
   po->Register("max-active-paths", &max_active_paths,
                "Used only when decoding_method is modified_beam_search");
 
+  po->Register("blank-penalty", &blank_penalty,
+               "The penalty applied on blank symbol during decoding. "
+               "Note: It is a positive value. "
+               "Increasing value will lead to lower deletion at the cost of higher insertions. "
+               "Currently only applicable for transducer models.");
+
   po->Register(
       "hotwords-file", &hotwords_file,
       "The file containing hotwords, one words/phrases per line, and for each"
@@ -74,7 +80,8 @@ std::string OfflineRecognizerConfig::ToString() const {
   os << "decoding_method=\"" << decoding_method << "\", ";
   os << "max_active_paths=" << max_active_paths << ", ";
   os << "hotwords_file=\"" << hotwords_file << "\", ";
-  os << "hotwords_score=" << hotwords_score << ")";
+  os << "hotwords_score=" << hotwords_score << ", ";
+  os << "blank_penalty=" << blank_penalty << ")";
 
   return os.str();
 }

sherpa-onnx/csrc/offline-recognizer.h

@@ -37,6 +37,8 @@ struct OfflineRecognizerConfig {
   std::string hotwords_file;
   float hotwords_score = 1.5;
 
+  float blank_penalty = 0.0;
+
   // only greedy_search is implemented
   // TODO(fangjun): Implement modified_beam_search
 
@@ -46,15 +48,17 @@ struct OfflineRecognizerConfig {
       const OfflineModelConfig &model_config, const OfflineLMConfig &lm_config,
       const OfflineCtcFstDecoderConfig &ctc_fst_decoder_config,
       const std::string &decoding_method, int32_t max_active_paths,
-      const std::string &hotwords_file, float hotwords_score)
+      const std::string &hotwords_file, float hotwords_score,
+      float blank_penalty)
       : feat_config(feat_config),
         model_config(model_config),
         lm_config(lm_config),
         ctc_fst_decoder_config(ctc_fst_decoder_config),
         decoding_method(decoding_method),
         max_active_paths(max_active_paths),
         hotwords_file(hotwords_file),
-        hotwords_score(hotwords_score) {}
+        hotwords_score(hotwords_score),
+        blank_penalty(blank_penalty) {}
 
   void Register(ParseOptions *po);
   bool Validate() const;

sherpa-onnx/csrc/offline-transducer-greedy-search-decoder.cc

@@ -46,7 +46,10 @@ OfflineTransducerGreedySearchDecoder::Decode(Ort::Value encoder_out,
     start += n;
     Ort::Value logit = model_->RunJoiner(std::move(cur_encoder_out),
                                          std::move(cur_decoder_out));
-    const float *p_logit = logit.GetTensorData<float>();
+    float *p_logit = logit.GetTensorMutableData<float>();
+    if (blank_penalty_ > 0.0) {
+      p_logit[0] -= blank_penalty_; // assuming blank id is 0
+    }
     bool emitted = false;
     for (int32_t i = 0; i != n; ++i) {
       auto y = static_cast<int32_t>(std::distance(

sherpa-onnx/csrc/offline-transducer-greedy-search-decoder.h

@@ -14,15 +14,18 @@ namespace sherpa_onnx {
 
 class OfflineTransducerGreedySearchDecoder : public OfflineTransducerDecoder {
  public:
-  explicit OfflineTransducerGreedySearchDecoder(OfflineTransducerModel *model)
-      : model_(model) {}
+  explicit OfflineTransducerGreedySearchDecoder(OfflineTransducerModel *model,
+                                                float blank_penalty)
+      : model_(model),
+        blank_penalty_(blank_penalty) {}
 
   std::vector<OfflineTransducerDecoderResult> Decode(
       Ort::Value encoder_out, Ort::Value encoder_out_length,
       OfflineStream **ss = nullptr, int32_t n = 0) override;
 
  private:
   OfflineTransducerModel *model_;  // Not owned
+  float blank_penalty_;
 };
 
 }  // namespace sherpa_onnx

sherpa-onnx/csrc/offline-transducer-modified-beam-search-decoder.cc

@@ -97,6 +97,10 @@ OfflineTransducerModifiedBeamSearchDecoder::Decode(
         model_->RunJoiner(std::move(cur_encoder_out), View(&decoder_out));
 
     float *p_logit = logit.GetTensorMutableData<float>();
+    if (blank_penalty_ > 0.0) {
+      SubtractBlank(p_logit, vocab_size, num_hyps, 0, blank_penalty_); // assuming blank id is 0
+      // p_logit[:, 0] -= blank_penalty_ // assuming blank id is 0
+    }
     LogSoftmax(p_logit, vocab_size, num_hyps);
 
     // now p_logit contains log_softmax output, we rename it to p_logprob

sherpa-onnx/csrc/offline-transducer-modified-beam-search-decoder.h

@@ -19,11 +19,13 @@ class OfflineTransducerModifiedBeamSearchDecoder
   OfflineTransducerModifiedBeamSearchDecoder(OfflineTransducerModel *model,
                                              OfflineLM *lm,
                                              int32_t max_active_paths,
-                                             float lm_scale)
+                                             float lm_scale,
+                                             float blank_penalty)
       : model_(model),
         lm_(lm),
         max_active_paths_(max_active_paths),
-        lm_scale_(lm_scale) {}
+        lm_scale_(lm_scale),
+        blank_penalty_(blank_penalty) {}
 
   std::vector<OfflineTransducerDecoderResult> Decode(
       Ort::Value encoder_out, Ort::Value encoder_out_length,
@@ -35,6 +37,7 @@ class OfflineTransducerModifiedBeamSearchDecoder
 
   int32_t max_active_paths_;
   float lm_scale_;  // used only when lm_ is not nullptr
+  float blank_penalty_;
 };
 
 }  // namespace sherpa_onnx

sherpa-onnx/python/csrc/offline-recognizer.cc

@@ -17,13 +17,14 @@ static void PybindOfflineRecognizerConfig(py::module *m) {
       .def(py::init<const OfflineFeatureExtractorConfig &,
                     const OfflineModelConfig &, const OfflineLMConfig &,
                     const OfflineCtcFstDecoderConfig &, const std::string &,
-                    int32_t, const std::string &, float>(),
+                    int32_t, const std::string &, float, float>(),
            py::arg("feat_config"), py::arg("model_config"),
            py::arg("lm_config") = OfflineLMConfig(),
            py::arg("ctc_fst_decoder_config") = OfflineCtcFstDecoderConfig(),
            py::arg("decoding_method") = "greedy_search",
            py::arg("max_active_paths") = 4, py::arg("hotwords_file") = "",
-           py::arg("hotwords_score") = 1.5)
+           py::arg("hotwords_score") = 1.5,
+           py::arg("blank_penalty") = 0.0)
       .def_readwrite("feat_config", &PyClass::feat_config)
       .def_readwrite("model_config", &PyClass::model_config)
       .def_readwrite("lm_config", &PyClass::lm_config)
@@ -32,6 +33,7 @@ static void PybindOfflineRecognizerConfig(py::module *m) {
       .def_readwrite("max_active_paths", &PyClass::max_active_paths)
       .def_readwrite("hotwords_file", &PyClass::hotwords_file)
       .def_readwrite("hotwords_score", &PyClass::hotwords_score)
+      .def_readwrite("blank_penalty", &PyClass::blank_penalty)
       .def("__str__", &PyClass::ToString);
 }