From 610ab7ea3d8a9bc8f952f598a465586bc94ae1c4 Mon Sep 17 00:00:00 2001 From: xiao Date: Sun, 8 Sep 2024 16:33:34 +0800 Subject: [PATCH 01/20] Allowing tokens and hotwords be initialized from buffered strings (rather than from external files) currently, only supported OnlineTransducer models --- c-api-examples/CMakeLists.txt | 3 + ...zipformer-buffered-tokens-hotwords-c-api.c | 655 ++++++++++++++++++ sherpa-onnx/c-api/c-api.cc | 4 + sherpa-onnx/c-api/c-api.h | 5 + sherpa-onnx/csrc/online-model-config.cc | 2 +- sherpa-onnx/csrc/online-model-config.h | 7 +- .../csrc/online-recognizer-transducer-impl.h | 21 +- .../online-recognizer-transducer-nemo-impl.h | 3 +- sherpa-onnx/csrc/online-recognizer.h | 7 +- sherpa-onnx/csrc/symbol-table.cc | 11 +- sherpa-onnx/csrc/symbol-table.h | 4 +- 11 files changed, 711 insertions(+), 11 deletions(-) create mode 100644 c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c diff --git a/c-api-examples/CMakeLists.txt b/c-api-examples/CMakeLists.txt index 49fb8fad7..3cac30001 100644 --- a/c-api-examples/CMakeLists.txt +++ b/c-api-examples/CMakeLists.txt @@ -48,6 +48,9 @@ target_link_libraries(telespeech-c-api sherpa-onnx-c-api) add_executable(vad-sense-voice-c-api vad-sense-voice-c-api.c) target_link_libraries(vad-sense-voice-c-api sherpa-onnx-c-api) +add_executable(streaming-zipformer-buffered-tokens-hotwords-c-api streaming-zipformer-buffered-tokens-hotwords-c-api.c) +target_link_libraries(streaming-zipformer-buffered-tokens-hotwords-c-api sherpa-onnx-c-api) + if(SHERPA_ONNX_HAS_ALSA) add_subdirectory(./asr-microphone-example) elseif((UNIX AND NOT APPLE) OR LINUX) diff --git a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c new file mode 100644 index 000000000..2fa27224c --- /dev/null +++ b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c @@ -0,0 +1,655 @@ +// c-api-examples/streaming-zipformer-c-api.c +// +// Copyright (c) 2024 Xiaomi Corporation + +// +// This file demonstrates how to use streaming Zipformer with sherpa-onnx's C and +// with tokens and hotwords loaded from buffered strings instread external files +// API. +// clang-format off +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2 +// tar xvf sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2 +// rm sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2 +// +// clang-format on + +#include +#include +#include + +#include "sherpa-onnx/c-api/c-api.h" + +extern const char* tokens_buf_str; +extern const char* hotwords_buf_str; +int32_t main() { + const char *wav_filename = + "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/test_wavs/0.wav"; + const char *encoder_filename = + "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/" + "encoder-epoch-99-avg-1.onnx"; + const char *decoder_filename = + "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/" + "decoder-epoch-99-avg-1.onnx"; + const char *joiner_filename = + "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/" + "joiner-epoch-99-avg-1.onnx"; + const char *provider = "cpu"; + const char *modeling_unit = "bpe"; + const char *bpe_vocab = + "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/" + "bpe.vocab"; + const SherpaOnnxWave *wave = SherpaOnnxReadWave(wav_filename); + if (wave == NULL) { + fprintf(stderr, "Failed to read %s\n", wav_filename); + return -1; + } + + // Zipformer config + SherpaOnnxOnlineTransducerModelConfig zipformer_config; + memset(&zipformer_config, 0, sizeof(zipformer_config)); + zipformer_config.encoder = encoder_filename; + zipformer_config.decoder = decoder_filename; + zipformer_config.joiner = joiner_filename; + + // Online model config + SherpaOnnxOnlineModelConfig online_model_config; + memset(&online_model_config, 0, sizeof(online_model_config)); + online_model_config.debug = 1; + online_model_config.num_threads = 1; + online_model_config.provider = provider; + online_model_config.tokens_buf_str = tokens_buf_str; + online_model_config.transducer = zipformer_config; + + // Recognizer config + SherpaOnnxOnlineRecognizerConfig recognizer_config; + memset(&recognizer_config, 0, sizeof(recognizer_config)); + recognizer_config.decoding_method = "modified_beam_search"; + recognizer_config.model_config = online_model_config; + recognizer_config.hotwords_buf_str = hotwords_buf_str; + + SherpaOnnxOnlineRecognizer *recognizer = + SherpaOnnxCreateOnlineRecognizer(&recognizer_config); + + if (recognizer == NULL) { + fprintf(stderr, "Please check your config!\n"); + SherpaOnnxFreeWave(wave); + return -1; + } + + SherpaOnnxOnlineStream *stream = SherpaOnnxCreateOnlineStream(recognizer); + + const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50); + int32_t segment_id = 0; + +// simulate streaming. You can choose an arbitrary N +#define N 3200 + + fprintf(stderr, "sample rate: %d, num samples: %d, duration: %.2f s\n", + wave->sample_rate, wave->num_samples, + (float)wave->num_samples / wave->sample_rate); + + int32_t k = 0; + while (k < wave->num_samples) { + int32_t start = k; + int32_t end = + (start + N > wave->num_samples) ? wave->num_samples : (start + N); + k += N; + + SherpaOnnxOnlineStreamAcceptWaveform(stream, wave->sample_rate, + wave->samples + start, end - start); + while (SherpaOnnxIsOnlineStreamReady(recognizer, stream)) { + SherpaOnnxDecodeOnlineStream(recognizer, stream); + } + + const SherpaOnnxOnlineRecognizerResult *r = + SherpaOnnxGetOnlineStreamResult(recognizer, stream); + + if (strlen(r->text)) { + SherpaOnnxPrint(display, segment_id, r->text); + } + + if (SherpaOnnxOnlineStreamIsEndpoint(recognizer, stream)) { + if (strlen(r->text)) { + ++segment_id; + } + SherpaOnnxOnlineStreamReset(recognizer, stream); + } + + SherpaOnnxDestroyOnlineRecognizerResult(r); + } + + // add some tail padding + float tail_paddings[4800] = {0}; // 0.3 seconds at 16 kHz sample rate + SherpaOnnxOnlineStreamAcceptWaveform(stream, wave->sample_rate, tail_paddings, + 4800); + + SherpaOnnxFreeWave(wave); + + SherpaOnnxOnlineStreamInputFinished(stream); + while (SherpaOnnxIsOnlineStreamReady(recognizer, stream)) { + SherpaOnnxDecodeOnlineStream(recognizer, stream); + } + + const SherpaOnnxOnlineRecognizerResult *r = + SherpaOnnxGetOnlineStreamResult(recognizer, stream); + + if (strlen(r->text)) { + SherpaOnnxPrint(display, segment_id, r->text); + } + + SherpaOnnxDestroyOnlineRecognizerResult(r); + + SherpaOnnxDestroyDisplay(display); + SherpaOnnxDestroyOnlineStream(stream); + SherpaOnnxDestroyOnlineRecognizer(recognizer); + fprintf(stderr, "\n"); + + return 0; +} + +const char* hotwords_buf_str = "▁A ▁T ▁P :1.5\n \ +▁A ▁B ▁C :3.0"; + +const char* tokens_buf_str = " 0 \n \ + 1 \n \ + 2 \n \ +S 3 \n \ +▁THE 4 \n \ +▁A 5 \n \ +T 6 \n \ +▁AND 7 \n \ +ED 8 \n \ +▁OF 9 \n \ +▁TO 10 \n \ +E 11 \n \ +D 12 \n \ +N 13 \n \ +ING 14 \n \ +▁IN 15 \n \ +Y 16 \n \ +M 17 \n \ +C 18 \n \ +▁I 19 \n \ +A 20 \n \ +P 21 \n \ +▁HE 22 \n \ +R 23 \n \ +O 24 \n \ +L 25 \n \ +RE 26 \n \ +I 27 \n \ +U 28 \n \ +ER 29 \n \ +▁IT 30 \n \ +LY 31 \n \ +▁THAT 32 \n \ +▁WAS 33 \n \ +▁ 34 \n \ +▁S 35 \n \ +AR 36 \n \ +▁BE 37 \n \ +F 38 \n \ +▁C 39 \n \ +IN 40 \n \ +B 41 \n \ +▁FOR 42 \n \ +OR 43 \n \ +LE 44 \n \ +' 45 \n \ +▁HIS 46 \n \ +▁YOU 47 \n \ +AL 48 \n \ +▁RE 49 \n \ +V 50 \n \ +▁B 51 \n \ +G 52 \n \ +RI 53 \n \ +▁E 54 \n \ +▁WITH 55 \n \ +▁T 56 \n \ +▁AS 57 \n \ +LL 58 \n \ +▁P 59 \n \ +▁HER 60 \n \ +ST 61 \n \ +▁HAD 62 \n \ +▁SO 63 \n \ +▁F 64 \n \ +W 65 \n \ +CE 66 \n \ +▁IS 67 \n \ +ND 68 \n \ +▁NOT 69 \n \ +TH 70 \n \ +▁BUT 71 \n \ +EN 72 \n \ +▁SHE 73 \n \ +▁ON 74 \n \ +VE 75 \n \ +ON 76 \n \ +SE 77 \n \ +▁DE 78 \n \ +UR 79 \n \ +▁G 80 \n \ +CH 81 \n \ +K 82 \n \ +TER 83 \n \ +▁AT 84 \n \ +IT 85 \n \ +▁ME 86 \n \ +RO 87 \n \ +NE 88 \n \ +RA 89 \n \ +ES 90 \n \ +IL 91 \n \ +NG 92 \n \ +IC 93 \n \ +▁NO 94 \n \ +▁HIM 95 \n \ +ENT 96 \n \ +IR 97 \n \ +▁WE 98 \n \ +H 99 \n \ +▁DO 100 \n \ +▁ALL 101 \n \ +▁HAVE 102 \n \ +LO 103 \n \ +▁BY 104 \n \ +▁MY 105 \n \ +▁MO 106 \n \ +▁THIS 107 \n \ +LA 108 \n \ +▁ST 109 \n \ +▁WHICH 110 \n \ +▁CON 111 \n \ +▁THEY 112 \n \ +CK 113 \n \ +TE 114 \n \ +▁SAID 115 \n \ +▁FROM 116 \n \ +▁GO 117 \n \ +▁WHO 118 \n \ +▁TH 119 \n \ +▁OR 120 \n \ +▁D 121 \n \ +▁W 122 \n \ +VER 123 \n \ +LI 124 \n \ +▁SE 125 \n \ +▁ONE 126 \n \ +▁CA 127 \n \ +▁AN 128 \n \ +▁LA 129 \n \ +▁WERE 130 \n \ +EL 131 \n \ +▁HA 132 \n \ +▁MAN 133 \n \ +▁FA 134 \n \ +▁EX 135 \n \ +AD 136 \n \ +▁SU 137 \n \ +RY 138 \n \ +▁MI 139 \n \ +AT 140 \n \ +▁BO 141 \n \ +▁WHEN 142 \n \ +AN 143 \n \ +THER 144 \n \ +PP 145 \n \ +ATION 146 \n \ +▁FI 147 \n \ +▁WOULD 148 \n \ +▁PRO 149 \n \ +OW 150 \n \ +ET 151 \n \ +▁O 152 \n \ +▁THERE 153 \n \ +▁HO 154 \n \ +ION 155 \n \ +▁WHAT 156 \n \ +▁FE 157 \n \ +▁PA 158 \n \ +US 159 \n \ +MENT 160 \n \ +▁MA 161 \n \ +UT 162 \n \ +▁OUT 163 \n \ +▁THEIR 164 \n \ +▁IF 165 \n \ +▁LI 166 \n \ +▁K 167 \n \ +▁WILL 168 \n \ +▁ARE 169 \n \ +ID 170 \n \ +▁RO 171 \n \ +DE 172 \n \ +TION 173 \n \ +▁WA 174 \n \ +PE 175 \n \ +▁UP 176 \n \ +▁SP 177 \n \ +▁PO 178 \n \ +IGHT 179 \n \ +▁UN 180 \n \ +RU 181 \n \ +▁LO 182 \n \ +AS 183 \n \ +OL 184 \n \ +▁LE 185 \n \ +▁BEEN 186 \n \ +▁SH 187 \n \ +▁RA 188 \n \ +▁SEE 189 \n \ +KE 190 \n \ +UL 191 \n \ +TED 192 \n \ +▁SA 193 \n \ +UN 194 \n \ +UND 195 \n \ +ANT 196 \n \ +▁NE 197 \n \ +IS 198 \n \ +▁THEM 199 \n \ +CI 200 \n \ +GE 201 \n \ +▁COULD 202 \n \ +▁DIS 203 \n \ +OM 204 \n \ +ISH 205 \n \ +HE 206 \n \ +EST 207 \n \ +▁SOME 208 \n \ +ENCE 209 \n \ +ITY 210 \n \ +IVE 211 \n \ +▁US 212 \n \ +▁MORE 213 \n \ +▁EN 214 \n \ +ARD 215 \n \ +ATE 216 \n \ +▁YOUR 217 \n \ +▁INTO 218 \n \ +▁KNOW 219 \n \ +▁CO 220 \n \ +ANCE 221 \n \ +▁TIME 222 \n \ +▁WI 223 \n \ +▁YE 224 \n \ +AGE 225 \n \ +▁NOW 226 \n \ +TI 227 \n \ +FF 228 \n \ +ABLE 229 \n \ +▁VERY 230 \n \ +▁LIKE 231 \n \ +AM 232 \n \ +HI 233 \n \ +Z 234 \n \ +▁OTHER 235 \n \ +▁THAN 236 \n \ +▁LITTLE 237 \n \ +▁DID 238 \n \ +▁LOOK 239 \n \ +TY 240 \n \ +ERS 241 \n \ +▁CAN 242 \n \ +▁CHA 243 \n \ +▁AR 244 \n \ +X 245 \n \ +FUL 246 \n \ +UGH 247 \n \ +▁BA 248 \n \ +▁DAY 249 \n \ +▁ABOUT 250 \n \ +TEN 251 \n \ +IM 252 \n \ +▁ANY 253 \n \ +▁PRE 254 \n \ +▁OVER 255 \n \ +IES 256 \n \ +NESS 257 \n \ +ME 258 \n \ +BLE 259 \n \ +▁M 260 \n \ +ROW 261 \n \ +▁HAS 262 \n \ +▁GREAT 263 \n \ +▁VI 264 \n \ +TA 265 \n \ +▁AFTER 266 \n \ +PER 267 \n \ +▁AGAIN 268 \n \ +HO 269 \n \ +SH 270 \n \ +▁UPON 271 \n \ +▁DI 272 \n \ +▁HAND 273 \n \ +▁COM 274 \n \ +IST 275 \n \ +TURE 276 \n \ +▁STA 277 \n \ +▁THEN 278 \n \ +▁SHOULD 279 \n \ +▁GA 280 \n \ +OUS 281 \n \ +OUR 282 \n \ +▁WELL 283 \n \ +▁ONLY 284 \n \ +MAN 285 \n \ +▁GOOD 286 \n \ +▁TWO 287 \n \ +▁MAR 288 \n \ +▁SAY 289 \n \ +▁HU 290 \n \ +TING 291 \n \ +▁OUR 292 \n \ +RESS 293 \n \ +▁DOWN 294 \n \ +IOUS 295 \n \ +▁BEFORE 296 \n \ +▁DA 297 \n \ +▁NA 298 \n \ +QUI 299 \n \ +▁MADE 300 \n \ +▁EVERY 301 \n \ +▁OLD 302 \n \ +▁EVEN 303 \n \ +IG 304 \n \ +▁COME 305 \n \ +▁GRA 306 \n \ +▁RI 307 \n \ +▁LONG 308 \n \ +OT 309 \n \ +SIDE 310 \n \ +WARD 311 \n \ +▁FO 312 \n \ +▁WHERE 313 \n \ +MO 314 \n \ +LESS 315 \n \ +▁SC 316 \n \ +▁MUST 317 \n \ +▁NEVER 318 \n \ +▁HOW 319 \n \ +▁CAME 320 \n \ +▁SUCH 321 \n \ +▁RU 322 \n \ +▁TAKE 323 \n \ +▁WO 324 \n \ +▁CAR 325 \n \ +UM 326 \n \ +AK 327 \n \ +▁THINK 328 \n \ +▁MUCH 329 \n \ +▁MISTER 330 \n \ +▁MAY 331 \n \ +▁JO 332 \n \ +▁WAY 333 \n \ +▁COMP 334 \n \ +▁THOUGHT 335 \n \ +▁STO 336 \n \ +▁MEN 337 \n \ +▁BACK 338 \n \ +▁DON 339 \n \ +J 340 \n \ +▁LET 341 \n \ +▁TRA 342 \n \ +▁FIRST 343 \n \ +▁JUST 344 \n \ +▁VA 345 \n \ +▁OWN 346 \n \ +▁PLA 347 \n \ +▁MAKE 348 \n \ +ATED 349 \n \ +▁HIMSELF 350 \n \ +▁WENT 351 \n \ +▁PI 352 \n \ +GG 353 \n \ +RING 354 \n \ +▁DU 355 \n \ +▁MIGHT 356 \n \ +▁PART 357 \n \ +▁GIVE 358 \n \ +▁IMP 359 \n \ +▁BU 360 \n \ +▁PER 361 \n \ +▁PLACE 362 \n \ +▁HOUSE 363 \n \ +▁THROUGH 364 \n \ +IAN 365 \n \ +▁SW 366 \n \ +▁UNDER 367 \n \ +QUE 368 \n \ +▁AWAY 369 \n \ +▁LOVE 370 \n \ +QUA 371 \n \ +▁LIFE 372 \n \ +▁GET 373 \n \ +▁WITHOUT 374 \n \ +▁PASS 375 \n \ +▁TURN 376 \n \ +IGN 377 \n \ +▁HEAD 378 \n \ +▁MOST 379 \n \ +▁THOSE 380 \n \ +▁SHALL 381 \n \ +▁EYES 382 \n \ +▁COL 383 \n \ +▁STILL 384 \n \ +▁NIGHT 385 \n \ +▁NOTHING 386 \n \ +ITION 387 \n \ +HA 388 \n \ +▁TELL 389 \n \ +▁WORK 390 \n \ +▁LAST 391 \n \ +▁NEW 392 \n \ +▁FACE 393 \n \ +▁HI 394 \n \ +▁WORD 395 \n \ +▁FOUND 396 \n \ +▁COUNT 397 \n \ +▁OB 398 \n \ +▁WHILE 399 \n \ +▁SHA 400 \n \ +▁MEAN 401 \n \ +▁SAW 402 \n \ +▁PEOPLE 403 \n \ +▁FRIEND 404 \n \ +▁THREE 405 \n \ +▁ROOM 406 \n \ +▁SAME 407 \n \ +▁THOUGH 408 \n \ +▁RIGHT 409 \n \ +▁CHILD 410 \n \ +▁FATHER 411 \n \ +▁ANOTHER 412 \n \ +▁HEART 413 \n \ +▁WANT 414 \n \ +▁TOOK 415 \n \ +OOK 416 \n \ +▁LIGHT 417 \n \ +▁MISSUS 418 \n \ +▁OPEN 419 \n \ +▁JU 420 \n \ +▁ASKED 421 \n \ +PORT 422 \n \ +▁LEFT 423 \n \ +▁JA 424 \n \ +▁WORLD 425 \n \ +▁HOME 426 \n \ +▁WHY 427 \n \ +▁ALWAYS 428 \n \ +▁ANSWER 429 \n \ +▁SEEMED 430 \n \ +▁SOMETHING 431 \n \ +▁GIRL 432 \n \ +▁BECAUSE 433 \n \ +▁NAME 434 \n \ +▁TOLD 435 \n \ +▁NI 436 \n \ +▁HIGH 437 \n \ +IZE 438 \n \ +▁WOMAN 439 \n \ +▁FOLLOW 440 \n \ +▁RETURN 441 \n \ +▁KNEW 442 \n \ +▁EACH 443 \n \ +▁KIND 444 \n \ +▁JE 445 \n \ +▁ACT 446 \n \ +▁LU 447 \n \ +▁CERTAIN 448 \n \ +▁YEARS 449 \n \ +▁QUITE 450 \n \ +▁APPEAR 451 \n \ +▁BETTER 452 \n \ +▁HALF 453 \n \ +▁PRESENT 454 \n \ +▁PRINCE 455 \n \ +SHIP 456 \n \ +▁ALSO 457 \n \ +▁BEGAN 458 \n \ +▁HAVING 459 \n \ +▁ENOUGH 460 \n \ +▁PERSON 461 \n \ +▁LADY 462 \n \ +▁WHITE 463 \n \ +▁COURSE 464 \n \ +▁VOICE 465 \n \ +▁SPEAK 466 \n \ +▁POWER 467 \n \ +▁MORNING 468 \n \ +▁BETWEEN 469 \n \ +▁AMONG 470 \n \ +▁KEEP 471 \n \ +▁WALK 472 \n \ +▁MATTER 473 \n \ +▁TEA 474 \n \ +▁BELIEVE 475 \n \ +▁SMALL 476 \n \ +▁TALK 477 \n \ +▁FELT 478 \n \ +▁HORSE 479 \n \ +▁MYSELF 480 \n \ +▁SIX 481 \n \ +▁HOWEVER 482 \n \ +▁FULL 483 \n \ +▁HERSELF 484 \n \ +▁POINT 485 \n \ +▁STOOD 486 \n \ +▁HUNDRED 487 \n \ +▁ALMOST 488 \n \ +▁SINCE 489 \n \ +▁LARGE 490 \n \ +▁LEAVE 491 \n \ +▁PERHAPS 492 \n \ +▁DARK 493 \n \ +▁SUDDEN 494 \n \ +▁REPLIED 495 \n \ +▁ANYTHING 496 \n \ +▁WONDER 497 \n \ +▁UNTIL 498 \n \ +Q 499 \n \ +#0 500 \n \ +#1 501"; \ No newline at end of file diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index cdee9a209..387875c77 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -73,6 +73,8 @@ SherpaOnnxOnlineRecognizer *SherpaOnnxCreateOnlineRecognizer( recognizer_config.model_config.tokens = SHERPA_ONNX_OR(config->model_config.tokens, ""); + recognizer_config.model_config.tokens_buf_str = + SHERPA_ONNX_OR(config->model_config.tokens_buf_str, ""); recognizer_config.model_config.num_threads = SHERPA_ONNX_OR(config->model_config.num_threads, 1); recognizer_config.model_config.provider_config.provider = @@ -118,6 +120,8 @@ SherpaOnnxOnlineRecognizer *SherpaOnnxCreateOnlineRecognizer( SHERPA_ONNX_OR(config->rule3_min_utterance_length, 20); recognizer_config.hotwords_file = SHERPA_ONNX_OR(config->hotwords_file, ""); + recognizer_config.hotwords_buf_str = + SHERPA_ONNX_OR(config->hotwords_buf_str, ""); recognizer_config.hotwords_score = SHERPA_ONNX_OR(config->hotwords_score, 1.5); diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index d4844aed1..053cab617 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -88,6 +88,8 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOnlineModelConfig { // - cjkchar+bpe const char *modeling_unit; const char *bpe_vocab; + /// if non-null, loading the tokens from the buffered string directly in prioriy + const char *tokens_buf_str; } SherpaOnnxOnlineModelConfig; /// It expects 16 kHz 16-bit single channel wave format. @@ -147,6 +149,9 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOnlineRecognizerConfig { const char *rule_fsts; const char *rule_fars; float blank_penalty; + + /// if non-nullptr, loading the hotwords from the buffered string directly in + const char *hotwords_buf_str; } SherpaOnnxOnlineRecognizerConfig; SHERPA_ONNX_API typedef struct SherpaOnnxOnlineRecognizerResult { diff --git a/sherpa-onnx/csrc/online-model-config.cc b/sherpa-onnx/csrc/online-model-config.cc index 9913fa9ed..ca5dc4b45 100644 --- a/sherpa-onnx/csrc/online-model-config.cc +++ b/sherpa-onnx/csrc/online-model-config.cc @@ -56,7 +56,7 @@ bool OnlineModelConfig::Validate() const { return false; } - if (!FileExists(tokens)) { + if (tokens_buf_str.empty() && !FileExists(tokens)) { SHERPA_ONNX_LOGE("tokens: '%s' does not exist", tokens.c_str()); return false; } diff --git a/sherpa-onnx/csrc/online-model-config.h b/sherpa-onnx/csrc/online-model-config.h index 0b64e06de..5e13c9cf5 100644 --- a/sherpa-onnx/csrc/online-model-config.h +++ b/sherpa-onnx/csrc/online-model-config.h @@ -45,6 +45,10 @@ struct OnlineModelConfig { std::string modeling_unit = "cjkchar"; std::string bpe_vocab; + /// if tokens_buf_str is non-empty, + /// the tokens will be loaded from the buffered string in prior to the ${tokens} file + std::string tokens_buf_str; + OnlineModelConfig() = default; OnlineModelConfig(const OnlineTransducerModelConfig &transducer, const OnlineParaformerModelConfig ¶former, @@ -69,7 +73,8 @@ struct OnlineModelConfig { debug(debug), model_type(model_type), modeling_unit(modeling_unit), - bpe_vocab(bpe_vocab) {} + bpe_vocab(bpe_vocab), + tokens_buf_str("") {} void Register(ParseOptions *po); bool Validate() const; diff --git a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h index ab1e165f3..e51e4d207 100644 --- a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h @@ -83,7 +83,8 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { : OnlineRecognizerImpl(config), config_(config), model_(OnlineTransducerModel::Create(config.model_config)), - sym_(config.model_config.tokens), + sym_(config.model_config.tokens_buf_str.empty() ? config.model_config.tokens : + config.model_config.tokens_buf_str, config.model_config.tokens_buf_str.empty() ? true : false), endpoint_(config_.endpoint_config) { if (sym_.Contains("")) { unk_id_ = sym_[""]; @@ -97,7 +98,9 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { config_.model_config.bpe_vocab); } - if (!config_.hotwords_file.empty()) { + if(!config_.hotwords_buf_str.empty()) { + InitHotwordsFromBufStr(); + } else if (!config_.hotwords_file.empty()) { InitHotwords(); } @@ -437,6 +440,20 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { } #endif + void InitHotwordsFromBufStr() { + // each line in hotwords_file contains space-separated words + + std::istringstream iss(config_.hotwords_buf_str); + if (!EncodeHotwords(iss, config_.model_config.modeling_unit, sym_, + bpe_encoder_.get(), &hotwords_, &boost_scores_)) { + SHERPA_ONNX_LOGE( + "Failed to encode some hotwords, skip them already, see logs above " + "for details."); + } + hotwords_graph_ = std::make_shared( + hotwords_, config_.hotwords_score, boost_scores_); + } + void InitOnlineStream(OnlineStream *stream) const { auto r = decoder_->GetEmptyResult(); diff --git a/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h b/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h index 700054dc2..37c150069 100644 --- a/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h @@ -44,7 +44,8 @@ class OnlineRecognizerTransducerNeMoImpl : public OnlineRecognizerImpl { const OnlineRecognizerConfig &config) : OnlineRecognizerImpl(config), config_(config), - symbol_table_(config.model_config.tokens), + symbol_table_(config.model_config.tokens_buf_str.empty() ? config.model_config.tokens : + config.model_config.tokens_buf_str, config.model_config.tokens_buf_str.empty() ? true : false), endpoint_(config_.endpoint_config), model_( std::make_unique(config.model_config)) { diff --git a/sherpa-onnx/csrc/online-recognizer.h b/sherpa-onnx/csrc/online-recognizer.h index 7fde367fb..f97fa43ae 100644 --- a/sherpa-onnx/csrc/online-recognizer.h +++ b/sherpa-onnx/csrc/online-recognizer.h @@ -106,6 +106,10 @@ struct OnlineRecognizerConfig { // If there are multiple FST archives, they are applied from left to right. std::string rule_fars; + /// used only for modified_beam_search, if hotwords_buf_str is non-empty, + /// the hotwords will be loaded from the buffered string in prior to the ${hotwords_file} + std::string hotwords_buf_str; + OnlineRecognizerConfig() = default; OnlineRecognizerConfig( @@ -130,7 +134,8 @@ struct OnlineRecognizerConfig { blank_penalty(blank_penalty), temperature_scale(temperature_scale), rule_fsts(rule_fsts), - rule_fars(rule_fars) {} + rule_fars(rule_fars), + hotwords_buf_str("") {} void Register(ParseOptions *po); bool Validate() const; diff --git a/sherpa-onnx/csrc/symbol-table.cc b/sherpa-onnx/csrc/symbol-table.cc index 8862972b7..5655c03a8 100644 --- a/sherpa-onnx/csrc/symbol-table.cc +++ b/sherpa-onnx/csrc/symbol-table.cc @@ -20,9 +20,14 @@ namespace sherpa_onnx { -SymbolTable::SymbolTable(const std::string &filename) { - std::ifstream is(filename); - Init(is); +SymbolTable::SymbolTable(const std::string &filename, bool is_file) { + if (is_file) { + std::ifstream is(filename); + Init(is); + } else { + std::istringstream iss(filename); + Init(iss); + } } #if __ANDROID_API__ >= 9 diff --git a/sherpa-onnx/csrc/symbol-table.h b/sherpa-onnx/csrc/symbol-table.h index 00d7a69e2..2c17b4d5e 100644 --- a/sherpa-onnx/csrc/symbol-table.h +++ b/sherpa-onnx/csrc/symbol-table.h @@ -19,13 +19,13 @@ namespace sherpa_onnx { class SymbolTable { public: SymbolTable() = default; - /// Construct a symbol table from a file. + /// Construct a symbol table from a file or from a buffered string. /// Each line in the file contains two fields: /// /// sym ID /// /// Fields are separated by space(s). - explicit SymbolTable(const std::string &filename); + explicit SymbolTable(const std::string &filename, bool is_file = true); #if __ANDROID_API__ >= 9 SymbolTable(AAssetManager *mgr, const std::string &filename); From ed4b2b551f7ac7e95faaafd36597f4ac2368ef4c Mon Sep 17 00:00:00 2001 From: xiao Date: Mon, 9 Sep 2024 18:23:56 +0800 Subject: [PATCH 02/20] add workflow c-api-test-loading-tokens-hotwords-from-memory.yaml, for test streaming models loading tokens and hotworks from buffered strings directly --- ...t-loading-tokens-hotwords-from-memory.yaml | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 .github/workflows/c-api-test-loading-tokens-hotwords-from-memory.yaml diff --git a/.github/workflows/c-api-test-loading-tokens-hotwords-from-memory.yaml b/.github/workflows/c-api-test-loading-tokens-hotwords-from-memory.yaml new file mode 100644 index 000000000..810ba1553 --- /dev/null +++ b/.github/workflows/c-api-test-loading-tokens-hotwords-from-memory.yaml @@ -0,0 +1,119 @@ +name: c-api-test-loading-tokens-hotwords-from-memory + +on: + push: + branches: + - master + tags: + - 'v[0-9]+.[0-9]+.[0-9]+*' + paths: + - '.github/workflows/c-api.yaml' + - 'CMakeLists.txt' + - 'cmake/**' + - 'sherpa-onnx/csrc/*' + - 'sherpa-onnx/c-api/*' + - 'c-api-examples/**' + - 'ffmpeg-examples/**' + pull_request: + branches: + - master + paths: + - '.github/workflows/c-api.yaml' + - 'CMakeLists.txt' + - 'cmake/**' + - 'sherpa-onnx/csrc/*' + - 'sherpa-onnx/c-api/*' + - 'c-api-examples/**' + - 'ffmpeg-examples/**' + + workflow_dispatch: + +concurrency: + group: c-api-${{ github.ref }} + cancel-in-progress: true + +jobs: + c_api: + name: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }}-c-api-shared + + - name: Build sherpa-onnx + shell: bash + run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + cmake --version + + mkdir build + cd build + + cmake \ + -D CMAKE_BUILD_TYPE=Release \ + -D BUILD_SHARED_LIBS=ON \ + -D CMAKE_INSTALL_PREFIX=./install \ + -D SHERPA_ONNX_ENABLE_BINARY=OFF \ + .. + + make -j2 install + + ls -lh install/lib + ls -lh install/include + + if [[ ${{ matrix.os }} == ubuntu-latest ]]; then + ldd ./install/lib/libsherpa-onnx-c-api.so + echo "---" + readelf -d ./install/lib/libsherpa-onnx-c-api.so + fi + + if [[ ${{ matrix.os }} == macos-latest ]]; then + otool -L ./install/lib/libsherpa-onnx-c-api.dylib + fi + + - name: Test streaming zipformer with tokens and hotwords loaded from buffers + shell: bash + run: | + gcc -o streaming-zipformer-buffered-tokens-hotwords-c-api ./c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + ls -lh streaming-zipformer-buffered-tokens-hotwords-c-api + + if [[ ${{ matrix.os }} == ubuntu-latest ]]; then + ldd ./streaming-zipformer-buffered-tokens-hotwords-c-api + echo "----" + readelf -d ./streaming-zipformer-buffered-tokens-hotwords-c-api + fi + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2 + tar xvf sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2 + rm sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2 + curl -SL -O https://huggingface.co/desh2608/icefall-asr-librispeech-pruned-transducer-stateless7-streaming-small/blob/main/data/lang_bpe_500/bpe.model + cp bpe.model sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/ + rm bpe.model + + ls -lh sherpa-onnx-streaming-zipformer-en-20M-2023-02-17 + echo "---" + ls -lh sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/test_wavs + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./streaming-zipformer-buffered-tokens-hotwords-c-api + + rm -rf sherpa-onnx-streaming-zipformer-* From 796819214cc49e76bfb635d58947cd2a2ba06276 Mon Sep 17 00:00:00 2001 From: lxiao336 Date: Tue, 10 Sep 2024 16:29:41 +0800 Subject: [PATCH 03/20] Update sherpa-onnx/csrc/online-model-config.h Co-authored-by: Fangjun Kuang --- sherpa-onnx/csrc/online-model-config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherpa-onnx/csrc/online-model-config.h b/sherpa-onnx/csrc/online-model-config.h index 5e13c9cf5..0861710e2 100644 --- a/sherpa-onnx/csrc/online-model-config.h +++ b/sherpa-onnx/csrc/online-model-config.h @@ -47,7 +47,7 @@ struct OnlineModelConfig { /// if tokens_buf_str is non-empty, /// the tokens will be loaded from the buffered string in prior to the ${tokens} file - std::string tokens_buf_str; + std::string tokens_buf; OnlineModelConfig() = default; OnlineModelConfig(const OnlineTransducerModelConfig &transducer, From 42b9a92c2cb2317e29602284a7a213cb7a414d7f Mon Sep 17 00:00:00 2001 From: lxiao336 Date: Tue, 10 Sep 2024 16:30:08 +0800 Subject: [PATCH 04/20] Update sherpa-onnx/csrc/online-model-config.h Co-authored-by: Fangjun Kuang --- sherpa-onnx/csrc/online-model-config.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sherpa-onnx/csrc/online-model-config.h b/sherpa-onnx/csrc/online-model-config.h index 0861710e2..b58651a5f 100644 --- a/sherpa-onnx/csrc/online-model-config.h +++ b/sherpa-onnx/csrc/online-model-config.h @@ -73,8 +73,7 @@ struct OnlineModelConfig { debug(debug), model_type(model_type), modeling_unit(modeling_unit), - bpe_vocab(bpe_vocab), - tokens_buf_str("") {} + bpe_vocab(bpe_vocab) {} void Register(ParseOptions *po); bool Validate() const; From 07a8cd7da1bfb360edd91d259d3c31a97865e77e Mon Sep 17 00:00:00 2001 From: xiao Date: Tue, 10 Sep 2024 18:01:32 +0800 Subject: [PATCH 05/20] 1. added token_buf_size and hotwords_buf_size to avoid memory overflow 2. rewrite some code to make it more readable 3. updated the c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c --- ...zipformer-buffered-tokens-hotwords-c-api.c | 60 ++++++++++++++++--- sherpa-onnx/c-api/c-api.cc | 19 ++++-- sherpa-onnx/c-api/c-api.h | 8 ++- sherpa-onnx/csrc/online-model-config.cc | 11 +++- sherpa-onnx/csrc/online-model-config.h | 2 +- .../csrc/online-recognizer-transducer-impl.h | 13 ++-- .../online-recognizer-transducer-nemo-impl.h | 9 ++- sherpa-onnx/csrc/online-recognizer.h | 7 +-- 8 files changed, 103 insertions(+), 26 deletions(-) diff --git a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c index 2fa27224c..05379602f 100644 --- a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c +++ b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c @@ -1,6 +1,7 @@ -// c-api-examples/streaming-zipformer-c-api.c +// c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c // // Copyright (c) 2024 Xiaomi Corporation +// Copyright (c) 2024 Luo Xiao // // This file demonstrates how to use streaming Zipformer with sherpa-onnx's C and @@ -20,8 +21,33 @@ #include "sherpa-onnx/c-api/c-api.h" -extern const char* tokens_buf_str; -extern const char* hotwords_buf_str; +size_t read_file(const char* filename, const char** buffer_out) { + FILE *file = fopen(filename, "rb"); + if (file == NULL) { + fprintf(stderr, "Failed to open %s\n", filename); + return -1; + } + fseek(file, 0L, SEEK_END); + long size = ftell(file); + rewind(file); + *buffer_out = malloc(size + 1); + if (*buffer_out == NULL) { + fclose(file); + fprintf(stderr, "Memory error\n"); + return -1; + } + size_t read_bytes = fread(*buffer_out, 1, size, file); + if (read_bytes != size) { + printf("Errors occured in reading the file %s\n", filename); + free(*buffer_out); + *buffer_out = NULL; + fclose(file); + return -1; + } + fclose(file); + return read_bytes; +} + int32_t main() { const char *wav_filename = "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/test_wavs/0.wav"; @@ -36,6 +62,8 @@ int32_t main() { "joiner-epoch-99-avg-1.onnx"; const char *provider = "cpu"; const char *modeling_unit = "bpe"; + const char *tokens_filename = "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/tokens.txt"; + const char *hotwords_filename = "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/hotwords.txt"; const char *bpe_vocab = "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/" "bpe.vocab"; @@ -45,6 +73,22 @@ int32_t main() { return -1; } + // reading tokens and hotwords to buffers + const *tokens_buf; + size_t token_buf_size = read_file(tokens_filename, &tokens_buf); + if(token_buf_size < 0) { + fprintf(stderr, "Please check your tokens.txt!\n"); + free(tokens_buf); + return -1; + } + const *hotwords_buf; + size_t hotwords_buf_size = read_file(hotwords_filename, &hotwords_buf); + if(hotwords_buf_size < 0) { + fprintf(stderr, "Please check your hotwords.txt!\n"); + free(hotwords_buf); + return -1; + } + // Zipformer config SherpaOnnxOnlineTransducerModelConfig zipformer_config; memset(&zipformer_config, 0, sizeof(zipformer_config)); @@ -58,7 +102,8 @@ int32_t main() { online_model_config.debug = 1; online_model_config.num_threads = 1; online_model_config.provider = provider; - online_model_config.tokens_buf_str = tokens_buf_str; + online_model_config.tokens_buf = tokens_buf; + online_model_config.tokens_buf_size = token_buf_size; online_model_config.transducer = zipformer_config; // Recognizer config @@ -66,7 +111,8 @@ int32_t main() { memset(&recognizer_config, 0, sizeof(recognizer_config)); recognizer_config.decoding_method = "modified_beam_search"; recognizer_config.model_config = online_model_config; - recognizer_config.hotwords_buf_str = hotwords_buf_str; + recognizer_config.hotwords_buf = hotwords_buf; + recognizer_config.hotwords_buf_size = hotwords_buf_size; SherpaOnnxOnlineRecognizer *recognizer = SherpaOnnxCreateOnlineRecognizer(&recognizer_config); @@ -148,10 +194,10 @@ int32_t main() { return 0; } -const char* hotwords_buf_str = "▁A ▁T ▁P :1.5\n \ +const char* hotwords_buf = "▁A ▁T ▁P :1.5\n \ ▁A ▁B ▁C :3.0"; -const char* tokens_buf_str = " 0 \n \ +const char* tokens_buf = " 0 \n \ 1 \n \ 2 \n \ S 3 \n \ diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 387875c77..d0706453f 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -73,8 +73,14 @@ SherpaOnnxOnlineRecognizer *SherpaOnnxCreateOnlineRecognizer( recognizer_config.model_config.tokens = SHERPA_ONNX_OR(config->model_config.tokens, ""); - recognizer_config.model_config.tokens_buf_str = - SHERPA_ONNX_OR(config->model_config.tokens_buf_str, ""); + if (config->model_config.tokens_buf && + config->model_config.tokens_buf_size > 0) { + recognizer_config.model_config.tokens_buf = std::string(config->model_config.tokens_buf, + config->model_config.tokens_buf_size); + } else { + recognizer_config.model_config.tokens_buf = ""; + } + recognizer_config.model_config.num_threads = SHERPA_ONNX_OR(config->model_config.num_threads, 1); recognizer_config.model_config.provider_config.provider = @@ -120,10 +126,15 @@ SherpaOnnxOnlineRecognizer *SherpaOnnxCreateOnlineRecognizer( SHERPA_ONNX_OR(config->rule3_min_utterance_length, 20); recognizer_config.hotwords_file = SHERPA_ONNX_OR(config->hotwords_file, ""); - recognizer_config.hotwords_buf_str = - SHERPA_ONNX_OR(config->hotwords_buf_str, ""); recognizer_config.hotwords_score = SHERPA_ONNX_OR(config->hotwords_score, 1.5); + if (config->hotwords_buf && + config->hotwords_buf_size > 0) { + recognizer_config.hotwords_buf = std::string(config->hotwords_buf, + config->hotwords_buf_size); + } else { + recognizer_config.hotwords_buf = ""; + } recognizer_config.blank_penalty = config->blank_penalty; diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 053cab617..82fc76e48 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -89,7 +89,9 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOnlineModelConfig { const char *modeling_unit; const char *bpe_vocab; /// if non-null, loading the tokens from the buffered string directly in prioriy - const char *tokens_buf_str; + const char *tokens_buf; + /// byte size excluding the tailing '\0' + int32_t tokens_buf_size; } SherpaOnnxOnlineModelConfig; /// It expects 16 kHz 16-bit single channel wave format. @@ -151,7 +153,9 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOnlineRecognizerConfig { float blank_penalty; /// if non-nullptr, loading the hotwords from the buffered string directly in - const char *hotwords_buf_str; + const char *hotwords_buf; + /// byte size excluding the tailing '\0' + int32_t hotwords_buf_size; } SherpaOnnxOnlineRecognizerConfig; SHERPA_ONNX_API typedef struct SherpaOnnxOnlineRecognizerResult { diff --git a/sherpa-onnx/csrc/online-model-config.cc b/sherpa-onnx/csrc/online-model-config.cc index ca5dc4b45..3f32ddc1d 100644 --- a/sherpa-onnx/csrc/online-model-config.cc +++ b/sherpa-onnx/csrc/online-model-config.cc @@ -56,8 +56,15 @@ bool OnlineModelConfig::Validate() const { return false; } - if (tokens_buf_str.empty() && !FileExists(tokens)) { - SHERPA_ONNX_LOGE("tokens: '%s' does not exist", tokens.c_str()); + if (!tokens_buf.empty() && FileExists(tokens)) { + SHERPA_ONNX_LOGE("you can not provide a tokens_buf and a tokens file: '%s', " + "at the same, which is confusing", tokens.c_str()); + return false; + } + + if (tokens_buf.empty() && !FileExists(tokens)) { + SHERPA_ONNX_LOGE("tokens: '%s' does not exist, you should provide " + "either a tokens buffer or a tokens file" , tokens.c_str()); return false; } diff --git a/sherpa-onnx/csrc/online-model-config.h b/sherpa-onnx/csrc/online-model-config.h index b58651a5f..ff1a0e228 100644 --- a/sherpa-onnx/csrc/online-model-config.h +++ b/sherpa-onnx/csrc/online-model-config.h @@ -45,7 +45,7 @@ struct OnlineModelConfig { std::string modeling_unit = "cjkchar"; std::string bpe_vocab; - /// if tokens_buf_str is non-empty, + /// if tokens_buf is non-empty, /// the tokens will be loaded from the buffered string in prior to the ${tokens} file std::string tokens_buf; diff --git a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h index e51e4d207..572a604ae 100644 --- a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h @@ -83,9 +83,14 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { : OnlineRecognizerImpl(config), config_(config), model_(OnlineTransducerModel::Create(config.model_config)), - sym_(config.model_config.tokens_buf_str.empty() ? config.model_config.tokens : - config.model_config.tokens_buf_str, config.model_config.tokens_buf_str.empty() ? true : false), endpoint_(config_.endpoint_config) { + if(!config.model_config.tokens_buf.empty()) { + sym_ = std::move(SymbolTable(config.model_config.tokens_buf, false)); + } else { + /// assuming tokens_buf and tokens are guaranteed not being both empty + sym_ = std::move(SymbolTable(config.model_config.tokens, true)); + } + if (sym_.Contains("")) { unk_id_ = sym_[""]; } @@ -98,7 +103,7 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { config_.model_config.bpe_vocab); } - if(!config_.hotwords_buf_str.empty()) { + if(!config_.hotwords_buf.empty()) { InitHotwordsFromBufStr(); } else if (!config_.hotwords_file.empty()) { InitHotwords(); @@ -443,7 +448,7 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { void InitHotwordsFromBufStr() { // each line in hotwords_file contains space-separated words - std::istringstream iss(config_.hotwords_buf_str); + std::istringstream iss(config_.hotwords_buf); if (!EncodeHotwords(iss, config_.model_config.modeling_unit, sym_, bpe_encoder_.get(), &hotwords_, &boost_scores_)) { SHERPA_ONNX_LOGE( diff --git a/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h b/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h index 37c150069..cc10fabe2 100644 --- a/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h @@ -44,11 +44,16 @@ class OnlineRecognizerTransducerNeMoImpl : public OnlineRecognizerImpl { const OnlineRecognizerConfig &config) : OnlineRecognizerImpl(config), config_(config), - symbol_table_(config.model_config.tokens_buf_str.empty() ? config.model_config.tokens : - config.model_config.tokens_buf_str, config.model_config.tokens_buf_str.empty() ? true : false), endpoint_(config_.endpoint_config), model_( std::make_unique(config.model_config)) { + if(!config.model_config.tokens_buf.empty()) { + symbol_table_ = std::move(SymbolTable(config.model_config.tokens_buf, false)); + } else { + /// assuming tokens_buf and tokens are guaranteed not being both empty + symbol_table_ = std::move(SymbolTable(config.model_config.tokens, true)); + } + if (config.decoding_method == "greedy_search") { decoder_ = std::make_unique( model_.get(), config_.blank_penalty); diff --git a/sherpa-onnx/csrc/online-recognizer.h b/sherpa-onnx/csrc/online-recognizer.h index f97fa43ae..7d71bc7fa 100644 --- a/sherpa-onnx/csrc/online-recognizer.h +++ b/sherpa-onnx/csrc/online-recognizer.h @@ -106,9 +106,9 @@ struct OnlineRecognizerConfig { // If there are multiple FST archives, they are applied from left to right. std::string rule_fars; - /// used only for modified_beam_search, if hotwords_buf_str is non-empty, + /// used only for modified_beam_search, if hotwords_buf is non-empty, /// the hotwords will be loaded from the buffered string in prior to the ${hotwords_file} - std::string hotwords_buf_str; + std::string hotwords_buf; OnlineRecognizerConfig() = default; @@ -134,8 +134,7 @@ struct OnlineRecognizerConfig { blank_penalty(blank_penalty), temperature_scale(temperature_scale), rule_fsts(rule_fsts), - rule_fars(rule_fars), - hotwords_buf_str("") {} + rule_fars(rule_fars) {} void Register(ParseOptions *po); bool Validate() const; From f309e1f94884fa85f1ee163aa6bb252676b77c0c Mon Sep 17 00:00:00 2001 From: xiao Date: Tue, 10 Sep 2024 20:42:07 +0800 Subject: [PATCH 06/20] 1. updated c-api-test-loading-tokens-hotwords-from-memory, tokens and hotwords are read from files before being filled into buffers 2. updated the test yaml --- ...t-loading-tokens-hotwords-from-memory.yaml | 2 + ...zipformer-buffered-tokens-hotwords-c-api.c | 512 +----------------- 2 files changed, 5 insertions(+), 509 deletions(-) diff --git a/.github/workflows/c-api-test-loading-tokens-hotwords-from-memory.yaml b/.github/workflows/c-api-test-loading-tokens-hotwords-from-memory.yaml index 810ba1553..6ce4372ba 100644 --- a/.github/workflows/c-api-test-loading-tokens-hotwords-from-memory.yaml +++ b/.github/workflows/c-api-test-loading-tokens-hotwords-from-memory.yaml @@ -107,6 +107,8 @@ jobs: cp bpe.model sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/ rm bpe.model + printf "▁A ▁T ▁P :1.5\n▁A ▁B ▁C :3.0" > hotwords.txt + ls -lh sherpa-onnx-streaming-zipformer-en-20M-2023-02-17 echo "---" ls -lh sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/test_wavs diff --git a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c index 05379602f..bd8b1296c 100644 --- a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c +++ b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c @@ -76,14 +76,14 @@ int32_t main() { // reading tokens and hotwords to buffers const *tokens_buf; size_t token_buf_size = read_file(tokens_filename, &tokens_buf); - if(token_buf_size < 0) { + if(token_buf_size < 1) { fprintf(stderr, "Please check your tokens.txt!\n"); free(tokens_buf); return -1; } const *hotwords_buf; size_t hotwords_buf_size = read_file(hotwords_filename, &hotwords_buf); - if(hotwords_buf_size < 0) { + if(hotwords_buf_size < 1) { fprintf(stderr, "Please check your hotwords.txt!\n"); free(hotwords_buf); return -1; @@ -192,510 +192,4 @@ int32_t main() { fprintf(stderr, "\n"); return 0; -} - -const char* hotwords_buf = "▁A ▁T ▁P :1.5\n \ -▁A ▁B ▁C :3.0"; - -const char* tokens_buf = " 0 \n \ - 1 \n \ - 2 \n \ -S 3 \n \ -▁THE 4 \n \ -▁A 5 \n \ -T 6 \n \ -▁AND 7 \n \ -ED 8 \n \ -▁OF 9 \n \ -▁TO 10 \n \ -E 11 \n \ -D 12 \n \ -N 13 \n \ -ING 14 \n \ -▁IN 15 \n \ -Y 16 \n \ -M 17 \n \ -C 18 \n \ -▁I 19 \n \ -A 20 \n \ -P 21 \n \ -▁HE 22 \n \ -R 23 \n \ -O 24 \n \ -L 25 \n \ -RE 26 \n \ -I 27 \n \ -U 28 \n \ -ER 29 \n \ -▁IT 30 \n \ -LY 31 \n \ -▁THAT 32 \n \ -▁WAS 33 \n \ -▁ 34 \n \ -▁S 35 \n \ -AR 36 \n \ -▁BE 37 \n \ -F 38 \n \ -▁C 39 \n \ -IN 40 \n \ -B 41 \n \ -▁FOR 42 \n \ -OR 43 \n \ -LE 44 \n \ -' 45 \n \ -▁HIS 46 \n \ -▁YOU 47 \n \ -AL 48 \n \ -▁RE 49 \n \ -V 50 \n \ -▁B 51 \n \ -G 52 \n \ -RI 53 \n \ -▁E 54 \n \ -▁WITH 55 \n \ -▁T 56 \n \ -▁AS 57 \n \ -LL 58 \n \ -▁P 59 \n \ -▁HER 60 \n \ -ST 61 \n \ -▁HAD 62 \n \ -▁SO 63 \n \ -▁F 64 \n \ -W 65 \n \ -CE 66 \n \ -▁IS 67 \n \ -ND 68 \n \ -▁NOT 69 \n \ -TH 70 \n \ -▁BUT 71 \n \ -EN 72 \n \ -▁SHE 73 \n \ -▁ON 74 \n \ -VE 75 \n \ -ON 76 \n \ -SE 77 \n \ -▁DE 78 \n \ -UR 79 \n \ -▁G 80 \n \ -CH 81 \n \ -K 82 \n \ -TER 83 \n \ -▁AT 84 \n \ -IT 85 \n \ -▁ME 86 \n \ -RO 87 \n \ -NE 88 \n \ -RA 89 \n \ -ES 90 \n \ -IL 91 \n \ -NG 92 \n \ -IC 93 \n \ -▁NO 94 \n \ -▁HIM 95 \n \ -ENT 96 \n \ -IR 97 \n \ -▁WE 98 \n \ -H 99 \n \ -▁DO 100 \n \ -▁ALL 101 \n \ -▁HAVE 102 \n \ -LO 103 \n \ -▁BY 104 \n \ -▁MY 105 \n \ -▁MO 106 \n \ -▁THIS 107 \n \ -LA 108 \n \ -▁ST 109 \n \ -▁WHICH 110 \n \ -▁CON 111 \n \ -▁THEY 112 \n \ -CK 113 \n \ -TE 114 \n \ -▁SAID 115 \n \ -▁FROM 116 \n \ -▁GO 117 \n \ -▁WHO 118 \n \ -▁TH 119 \n \ -▁OR 120 \n \ -▁D 121 \n \ -▁W 122 \n \ -VER 123 \n \ -LI 124 \n \ -▁SE 125 \n \ -▁ONE 126 \n \ -▁CA 127 \n \ -▁AN 128 \n \ -▁LA 129 \n \ -▁WERE 130 \n \ -EL 131 \n \ -▁HA 132 \n \ -▁MAN 133 \n \ -▁FA 134 \n \ -▁EX 135 \n \ -AD 136 \n \ -▁SU 137 \n \ -RY 138 \n \ -▁MI 139 \n \ -AT 140 \n \ -▁BO 141 \n \ -▁WHEN 142 \n \ -AN 143 \n \ -THER 144 \n \ -PP 145 \n \ -ATION 146 \n \ -▁FI 147 \n \ -▁WOULD 148 \n \ -▁PRO 149 \n \ -OW 150 \n \ -ET 151 \n \ -▁O 152 \n \ -▁THERE 153 \n \ -▁HO 154 \n \ -ION 155 \n \ -▁WHAT 156 \n \ -▁FE 157 \n \ -▁PA 158 \n \ -US 159 \n \ -MENT 160 \n \ -▁MA 161 \n \ -UT 162 \n \ -▁OUT 163 \n \ -▁THEIR 164 \n \ -▁IF 165 \n \ -▁LI 166 \n \ -▁K 167 \n \ -▁WILL 168 \n \ -▁ARE 169 \n \ -ID 170 \n \ -▁RO 171 \n \ -DE 172 \n \ -TION 173 \n \ -▁WA 174 \n \ -PE 175 \n \ -▁UP 176 \n \ -▁SP 177 \n \ -▁PO 178 \n \ -IGHT 179 \n \ -▁UN 180 \n \ -RU 181 \n \ -▁LO 182 \n \ -AS 183 \n \ -OL 184 \n \ -▁LE 185 \n \ -▁BEEN 186 \n \ -▁SH 187 \n \ -▁RA 188 \n \ -▁SEE 189 \n \ -KE 190 \n \ -UL 191 \n \ -TED 192 \n \ -▁SA 193 \n \ -UN 194 \n \ -UND 195 \n \ -ANT 196 \n \ -▁NE 197 \n \ -IS 198 \n \ -▁THEM 199 \n \ -CI 200 \n \ -GE 201 \n \ -▁COULD 202 \n \ -▁DIS 203 \n \ -OM 204 \n \ -ISH 205 \n \ -HE 206 \n \ -EST 207 \n \ -▁SOME 208 \n \ -ENCE 209 \n \ -ITY 210 \n \ -IVE 211 \n \ -▁US 212 \n \ -▁MORE 213 \n \ -▁EN 214 \n \ -ARD 215 \n \ -ATE 216 \n \ -▁YOUR 217 \n \ -▁INTO 218 \n \ -▁KNOW 219 \n \ -▁CO 220 \n \ -ANCE 221 \n \ -▁TIME 222 \n \ -▁WI 223 \n \ -▁YE 224 \n \ -AGE 225 \n \ -▁NOW 226 \n \ -TI 227 \n \ -FF 228 \n \ -ABLE 229 \n \ -▁VERY 230 \n \ -▁LIKE 231 \n \ -AM 232 \n \ -HI 233 \n \ -Z 234 \n \ -▁OTHER 235 \n \ -▁THAN 236 \n \ -▁LITTLE 237 \n \ -▁DID 238 \n \ -▁LOOK 239 \n \ -TY 240 \n \ -ERS 241 \n \ -▁CAN 242 \n \ -▁CHA 243 \n \ -▁AR 244 \n \ -X 245 \n \ -FUL 246 \n \ -UGH 247 \n \ -▁BA 248 \n \ -▁DAY 249 \n \ -▁ABOUT 250 \n \ -TEN 251 \n \ -IM 252 \n \ -▁ANY 253 \n \ -▁PRE 254 \n \ -▁OVER 255 \n \ -IES 256 \n \ -NESS 257 \n \ -ME 258 \n \ -BLE 259 \n \ -▁M 260 \n \ -ROW 261 \n \ -▁HAS 262 \n \ -▁GREAT 263 \n \ -▁VI 264 \n \ -TA 265 \n \ -▁AFTER 266 \n \ -PER 267 \n \ -▁AGAIN 268 \n \ -HO 269 \n \ -SH 270 \n \ -▁UPON 271 \n \ -▁DI 272 \n \ -▁HAND 273 \n \ -▁COM 274 \n \ -IST 275 \n \ -TURE 276 \n \ -▁STA 277 \n \ -▁THEN 278 \n \ -▁SHOULD 279 \n \ -▁GA 280 \n \ -OUS 281 \n \ -OUR 282 \n \ -▁WELL 283 \n \ -▁ONLY 284 \n \ -MAN 285 \n \ -▁GOOD 286 \n \ -▁TWO 287 \n \ -▁MAR 288 \n \ -▁SAY 289 \n \ -▁HU 290 \n \ -TING 291 \n \ -▁OUR 292 \n \ -RESS 293 \n \ -▁DOWN 294 \n \ -IOUS 295 \n \ -▁BEFORE 296 \n \ -▁DA 297 \n \ -▁NA 298 \n \ -QUI 299 \n \ -▁MADE 300 \n \ -▁EVERY 301 \n \ -▁OLD 302 \n \ -▁EVEN 303 \n \ -IG 304 \n \ -▁COME 305 \n \ -▁GRA 306 \n \ -▁RI 307 \n \ -▁LONG 308 \n \ -OT 309 \n \ -SIDE 310 \n \ -WARD 311 \n \ -▁FO 312 \n \ -▁WHERE 313 \n \ -MO 314 \n \ -LESS 315 \n \ -▁SC 316 \n \ -▁MUST 317 \n \ -▁NEVER 318 \n \ -▁HOW 319 \n \ -▁CAME 320 \n \ -▁SUCH 321 \n \ -▁RU 322 \n \ -▁TAKE 323 \n \ -▁WO 324 \n \ -▁CAR 325 \n \ -UM 326 \n \ -AK 327 \n \ -▁THINK 328 \n \ -▁MUCH 329 \n \ -▁MISTER 330 \n \ -▁MAY 331 \n \ -▁JO 332 \n \ -▁WAY 333 \n \ -▁COMP 334 \n \ -▁THOUGHT 335 \n \ -▁STO 336 \n \ -▁MEN 337 \n \ -▁BACK 338 \n \ -▁DON 339 \n \ -J 340 \n \ -▁LET 341 \n \ -▁TRA 342 \n \ -▁FIRST 343 \n \ -▁JUST 344 \n \ -▁VA 345 \n \ -▁OWN 346 \n \ -▁PLA 347 \n \ -▁MAKE 348 \n \ -ATED 349 \n \ -▁HIMSELF 350 \n \ -▁WENT 351 \n \ -▁PI 352 \n \ -GG 353 \n \ -RING 354 \n \ -▁DU 355 \n \ -▁MIGHT 356 \n \ -▁PART 357 \n \ -▁GIVE 358 \n \ -▁IMP 359 \n \ -▁BU 360 \n \ -▁PER 361 \n \ -▁PLACE 362 \n \ -▁HOUSE 363 \n \ -▁THROUGH 364 \n \ -IAN 365 \n \ -▁SW 366 \n \ -▁UNDER 367 \n \ -QUE 368 \n \ -▁AWAY 369 \n \ -▁LOVE 370 \n \ -QUA 371 \n \ -▁LIFE 372 \n \ -▁GET 373 \n \ -▁WITHOUT 374 \n \ -▁PASS 375 \n \ -▁TURN 376 \n \ -IGN 377 \n \ -▁HEAD 378 \n \ -▁MOST 379 \n \ -▁THOSE 380 \n \ -▁SHALL 381 \n \ -▁EYES 382 \n \ -▁COL 383 \n \ -▁STILL 384 \n \ -▁NIGHT 385 \n \ -▁NOTHING 386 \n \ -ITION 387 \n \ -HA 388 \n \ -▁TELL 389 \n \ -▁WORK 390 \n \ -▁LAST 391 \n \ -▁NEW 392 \n \ -▁FACE 393 \n \ -▁HI 394 \n \ -▁WORD 395 \n \ -▁FOUND 396 \n \ -▁COUNT 397 \n \ -▁OB 398 \n \ -▁WHILE 399 \n \ -▁SHA 400 \n \ -▁MEAN 401 \n \ -▁SAW 402 \n \ -▁PEOPLE 403 \n \ -▁FRIEND 404 \n \ -▁THREE 405 \n \ -▁ROOM 406 \n \ -▁SAME 407 \n \ -▁THOUGH 408 \n \ -▁RIGHT 409 \n \ -▁CHILD 410 \n \ -▁FATHER 411 \n \ -▁ANOTHER 412 \n \ -▁HEART 413 \n \ -▁WANT 414 \n \ -▁TOOK 415 \n \ -OOK 416 \n \ -▁LIGHT 417 \n \ -▁MISSUS 418 \n \ -▁OPEN 419 \n \ -▁JU 420 \n \ -▁ASKED 421 \n \ -PORT 422 \n \ -▁LEFT 423 \n \ -▁JA 424 \n \ -▁WORLD 425 \n \ -▁HOME 426 \n \ -▁WHY 427 \n \ -▁ALWAYS 428 \n \ -▁ANSWER 429 \n \ -▁SEEMED 430 \n \ -▁SOMETHING 431 \n \ -▁GIRL 432 \n \ -▁BECAUSE 433 \n \ -▁NAME 434 \n \ -▁TOLD 435 \n \ -▁NI 436 \n \ -▁HIGH 437 \n \ -IZE 438 \n \ -▁WOMAN 439 \n \ -▁FOLLOW 440 \n \ -▁RETURN 441 \n \ -▁KNEW 442 \n \ -▁EACH 443 \n \ -▁KIND 444 \n \ -▁JE 445 \n \ -▁ACT 446 \n \ -▁LU 447 \n \ -▁CERTAIN 448 \n \ -▁YEARS 449 \n \ -▁QUITE 450 \n \ -▁APPEAR 451 \n \ -▁BETTER 452 \n \ -▁HALF 453 \n \ -▁PRESENT 454 \n \ -▁PRINCE 455 \n \ -SHIP 456 \n \ -▁ALSO 457 \n \ -▁BEGAN 458 \n \ -▁HAVING 459 \n \ -▁ENOUGH 460 \n \ -▁PERSON 461 \n \ -▁LADY 462 \n \ -▁WHITE 463 \n \ -▁COURSE 464 \n \ -▁VOICE 465 \n \ -▁SPEAK 466 \n \ -▁POWER 467 \n \ -▁MORNING 468 \n \ -▁BETWEEN 469 \n \ -▁AMONG 470 \n \ -▁KEEP 471 \n \ -▁WALK 472 \n \ -▁MATTER 473 \n \ -▁TEA 474 \n \ -▁BELIEVE 475 \n \ -▁SMALL 476 \n \ -▁TALK 477 \n \ -▁FELT 478 \n \ -▁HORSE 479 \n \ -▁MYSELF 480 \n \ -▁SIX 481 \n \ -▁HOWEVER 482 \n \ -▁FULL 483 \n \ -▁HERSELF 484 \n \ -▁POINT 485 \n \ -▁STOOD 486 \n \ -▁HUNDRED 487 \n \ -▁ALMOST 488 \n \ -▁SINCE 489 \n \ -▁LARGE 490 \n \ -▁LEAVE 491 \n \ -▁PERHAPS 492 \n \ -▁DARK 493 \n \ -▁SUDDEN 494 \n \ -▁REPLIED 495 \n \ -▁ANYTHING 496 \n \ -▁WONDER 497 \n \ -▁UNTIL 498 \n \ -Q 499 \n \ -#0 500 \n \ -#1 501"; \ No newline at end of file +} \ No newline at end of file From e5dd86e6dc998ee08a87ee7bf595c9ad2746306a Mon Sep 17 00:00:00 2001 From: xiao Date: Wed, 11 Sep 2024 09:02:30 +0800 Subject: [PATCH 07/20] corrected const *hotwords_buf const* tokens_buf to const char* --- .../streaming-zipformer-buffered-tokens-hotwords-c-api.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c index bd8b1296c..fa933404a 100644 --- a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c +++ b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c @@ -74,14 +74,14 @@ int32_t main() { } // reading tokens and hotwords to buffers - const *tokens_buf; + const char *tokens_buf; size_t token_buf_size = read_file(tokens_filename, &tokens_buf); if(token_buf_size < 1) { fprintf(stderr, "Please check your tokens.txt!\n"); free(tokens_buf); return -1; } - const *hotwords_buf; + const char *hotwords_buf; size_t hotwords_buf_size = read_file(hotwords_filename, &hotwords_buf); if(hotwords_buf_size < 1) { fprintf(stderr, "Please check your hotwords.txt!\n"); From 12540233b62755437f7390dba8910bd3fba42ec6 Mon Sep 17 00:00:00 2001 From: xiao Date: Wed, 11 Sep 2024 21:35:56 +0800 Subject: [PATCH 08/20] formated long lines with clang-format --- sherpa-onnx/c-api/c-api.cc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index d0706453f..dfaeaa616 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -75,8 +75,8 @@ SherpaOnnxOnlineRecognizer *SherpaOnnxCreateOnlineRecognizer( SHERPA_ONNX_OR(config->model_config.tokens, ""); if (config->model_config.tokens_buf && config->model_config.tokens_buf_size > 0) { - recognizer_config.model_config.tokens_buf = std::string(config->model_config.tokens_buf, - config->model_config.tokens_buf_size); + recognizer_config.model_config.tokens_buf = std::string( + config->model_config.tokens_buf, config->model_config.tokens_buf_size); } else { recognizer_config.model_config.tokens_buf = ""; } @@ -128,10 +128,9 @@ SherpaOnnxOnlineRecognizer *SherpaOnnxCreateOnlineRecognizer( recognizer_config.hotwords_file = SHERPA_ONNX_OR(config->hotwords_file, ""); recognizer_config.hotwords_score = SHERPA_ONNX_OR(config->hotwords_score, 1.5); - if (config->hotwords_buf && - config->hotwords_buf_size > 0) { - recognizer_config.hotwords_buf = std::string(config->hotwords_buf, - config->hotwords_buf_size); + if (config->hotwords_buf && config->hotwords_buf_size > 0) { + recognizer_config.hotwords_buf = + std::string(config->hotwords_buf, config->hotwords_buf_size); } else { recognizer_config.hotwords_buf = ""; } From da14d2289151213828faf87ae73bd1a980a0e525 Mon Sep 17 00:00:00 2001 From: xiao Date: Thu, 12 Sep 2024 09:41:48 +0800 Subject: [PATCH 09/20] applied clang-format to all modified files --- c-api-examples/CMakeLists.txt | 3 +- ...zipformer-buffered-tokens-hotwords-c-api.c | 30 ++++++++++--------- sherpa-onnx/c-api/c-api.h | 7 +++-- sherpa-onnx/csrc/online-model-config.cc | 12 +++++--- sherpa-onnx/csrc/online-model-config.h | 8 ++--- .../csrc/online-recognizer-transducer-impl.h | 16 +++++----- .../online-recognizer-transducer-nemo-impl.h | 7 +++-- sherpa-onnx/csrc/online-recognizer.h | 3 +- 8 files changed, 47 insertions(+), 39 deletions(-) diff --git a/c-api-examples/CMakeLists.txt b/c-api-examples/CMakeLists.txt index 3cac30001..954487f9f 100644 --- a/c-api-examples/CMakeLists.txt +++ b/c-api-examples/CMakeLists.txt @@ -48,7 +48,8 @@ target_link_libraries(telespeech-c-api sherpa-onnx-c-api) add_executable(vad-sense-voice-c-api vad-sense-voice-c-api.c) target_link_libraries(vad-sense-voice-c-api sherpa-onnx-c-api) -add_executable(streaming-zipformer-buffered-tokens-hotwords-c-api streaming-zipformer-buffered-tokens-hotwords-c-api.c) +add_executable(streaming-zipformer-buffered-tokens-hotwords-c-api + streaming-zipformer-buffered-tokens-hotwords-c-api.c) target_link_libraries(streaming-zipformer-buffered-tokens-hotwords-c-api sherpa-onnx-c-api) if(SHERPA_ONNX_HAS_ALSA) diff --git a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c index fa933404a..c9ee14f49 100644 --- a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c +++ b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c @@ -4,9 +4,9 @@ // Copyright (c) 2024 Luo Xiao // -// This file demonstrates how to use streaming Zipformer with sherpa-onnx's C and -// with tokens and hotwords loaded from buffered strings instread external files -// API. +// This file demonstrates how to use streaming Zipformer with sherpa-onnx's C +// and with tokens and hotwords loaded from buffered strings instread external +// files API. // clang-format off // // wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2 @@ -21,7 +21,7 @@ #include "sherpa-onnx/c-api/c-api.h" -size_t read_file(const char* filename, const char** buffer_out) { +size_t read_file(const char *filename, const char **buffer_out) { FILE *file = fopen(filename, "rb"); if (file == NULL) { fprintf(stderr, "Failed to open %s\n", filename); @@ -38,11 +38,11 @@ size_t read_file(const char* filename, const char** buffer_out) { } size_t read_bytes = fread(*buffer_out, 1, size, file); if (read_bytes != size) { - printf("Errors occured in reading the file %s\n", filename); - free(*buffer_out); - *buffer_out = NULL; - fclose(file); - return -1; + printf("Errors occured in reading the file %s\n", filename); + free(*buffer_out); + *buffer_out = NULL; + fclose(file); + return -1; } fclose(file); return read_bytes; @@ -62,9 +62,11 @@ int32_t main() { "joiner-epoch-99-avg-1.onnx"; const char *provider = "cpu"; const char *modeling_unit = "bpe"; - const char *tokens_filename = "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/tokens.txt"; - const char *hotwords_filename = "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/hotwords.txt"; - const char *bpe_vocab = + const char *tokens_filename = + "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/tokens.txt"; + const char *hotwords_filename = + "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/hotwords.txt"; + const char *bpe_vocab = "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17/" "bpe.vocab"; const SherpaOnnxWave *wave = SherpaOnnxReadWave(wav_filename); @@ -76,14 +78,14 @@ int32_t main() { // reading tokens and hotwords to buffers const char *tokens_buf; size_t token_buf_size = read_file(tokens_filename, &tokens_buf); - if(token_buf_size < 1) { + if (token_buf_size < 1) { fprintf(stderr, "Please check your tokens.txt!\n"); free(tokens_buf); return -1; } const char *hotwords_buf; size_t hotwords_buf_size = read_file(hotwords_filename, &hotwords_buf); - if(hotwords_buf_size < 1) { + if (hotwords_buf_size < 1) { fprintf(stderr, "Please check your hotwords.txt!\n"); free(hotwords_buf); return -1; diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 82fc76e48..11dba9816 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -88,8 +88,9 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOnlineModelConfig { // - cjkchar+bpe const char *modeling_unit; const char *bpe_vocab; - /// if non-null, loading the tokens from the buffered string directly in prioriy - const char *tokens_buf; + /// if non-null, loading the tokens from the buffered string directly in + /// prioriy + const char *tokens_buf; /// byte size excluding the tailing '\0' int32_t tokens_buf_size; } SherpaOnnxOnlineModelConfig; @@ -151,7 +152,7 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOnlineRecognizerConfig { const char *rule_fsts; const char *rule_fars; float blank_penalty; - + /// if non-nullptr, loading the hotwords from the buffered string directly in const char *hotwords_buf; /// byte size excluding the tailing '\0' diff --git a/sherpa-onnx/csrc/online-model-config.cc b/sherpa-onnx/csrc/online-model-config.cc index 3f32ddc1d..efb5526dd 100644 --- a/sherpa-onnx/csrc/online-model-config.cc +++ b/sherpa-onnx/csrc/online-model-config.cc @@ -57,14 +57,18 @@ bool OnlineModelConfig::Validate() const { } if (!tokens_buf.empty() && FileExists(tokens)) { - SHERPA_ONNX_LOGE("you can not provide a tokens_buf and a tokens file: '%s', " - "at the same, which is confusing", tokens.c_str()); + SHERPA_ONNX_LOGE( + "you can not provide a tokens_buf and a tokens file: '%s', " + "at the same, which is confusing", + tokens.c_str()); return false; } if (tokens_buf.empty() && !FileExists(tokens)) { - SHERPA_ONNX_LOGE("tokens: '%s' does not exist, you should provide " - "either a tokens buffer or a tokens file" , tokens.c_str()); + SHERPA_ONNX_LOGE( + "tokens: '%s' does not exist, you should provide " + "either a tokens buffer or a tokens file", + tokens.c_str()); return false; } diff --git a/sherpa-onnx/csrc/online-model-config.h b/sherpa-onnx/csrc/online-model-config.h index ff1a0e228..127b8d454 100644 --- a/sherpa-onnx/csrc/online-model-config.h +++ b/sherpa-onnx/csrc/online-model-config.h @@ -45,8 +45,9 @@ struct OnlineModelConfig { std::string modeling_unit = "cjkchar"; std::string bpe_vocab; - /// if tokens_buf is non-empty, - /// the tokens will be loaded from the buffered string in prior to the ${tokens} file + /// if tokens_buf is non-empty, + /// the tokens will be loaded from the buffered string in prior to the + /// ${tokens} file std::string tokens_buf; OnlineModelConfig() = default; @@ -57,8 +58,7 @@ struct OnlineModelConfig { const OnlineNeMoCtcModelConfig &nemo_ctc, const ProviderConfig &provider_config, const std::string &tokens, int32_t num_threads, - int32_t warm_up, bool debug, - const std::string &model_type, + int32_t warm_up, bool debug, const std::string &model_type, const std::string &modeling_unit, const std::string &bpe_vocab) : transducer(transducer), diff --git a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h index 572a604ae..d1ce240bf 100644 --- a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h @@ -84,13 +84,13 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { config_(config), model_(OnlineTransducerModel::Create(config.model_config)), endpoint_(config_.endpoint_config) { - if(!config.model_config.tokens_buf.empty()) { + if (!config.model_config.tokens_buf.empty()) { sym_ = std::move(SymbolTable(config.model_config.tokens_buf, false)); } else { /// assuming tokens_buf and tokens are guaranteed not being both empty sym_ = std::move(SymbolTable(config.model_config.tokens, true)); - } - + } + if (sym_.Contains("")) { unk_id_ = sym_[""]; } @@ -103,7 +103,7 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { config_.model_config.bpe_vocab); } - if(!config_.hotwords_buf.empty()) { + if (!config_.hotwords_buf.empty()) { InitHotwordsFromBufStr(); } else if (!config_.hotwords_file.empty()) { InitHotwords(); @@ -116,8 +116,7 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { decoder_ = std::make_unique( model_.get(), lm_.get(), config_.max_active_paths, config_.lm_config.scale, config_.lm_config.shallow_fusion, unk_id_, - config_.blank_penalty, - config_.temperature_scale); + config_.blank_penalty, config_.temperature_scale); } else if (config.decoding_method == "greedy_search") { decoder_ = std::make_unique( @@ -166,8 +165,7 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { decoder_ = std::make_unique( model_.get(), lm_.get(), config_.max_active_paths, config_.lm_config.scale, config_.lm_config.shallow_fusion, unk_id_, - config_.blank_penalty, - config_.temperature_scale); + config_.blank_penalty, config_.temperature_scale); } else if (config.decoding_method == "greedy_search") { decoder_ = std::make_unique( @@ -458,7 +456,7 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { hotwords_graph_ = std::make_shared( hotwords_, config_.hotwords_score, boost_scores_); } - + void InitOnlineStream(OnlineStream *stream) const { auto r = decoder_->GetEmptyResult(); diff --git a/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h b/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h index cc10fabe2..866d7cc2b 100644 --- a/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h @@ -47,12 +47,13 @@ class OnlineRecognizerTransducerNeMoImpl : public OnlineRecognizerImpl { endpoint_(config_.endpoint_config), model_( std::make_unique(config.model_config)) { - if(!config.model_config.tokens_buf.empty()) { - symbol_table_ = std::move(SymbolTable(config.model_config.tokens_buf, false)); + if (!config.model_config.tokens_buf.empty()) { + symbol_table_ = + std::move(SymbolTable(config.model_config.tokens_buf, false)); } else { /// assuming tokens_buf and tokens are guaranteed not being both empty symbol_table_ = std::move(SymbolTable(config.model_config.tokens, true)); - } + } if (config.decoding_method == "greedy_search") { decoder_ = std::make_unique( diff --git a/sherpa-onnx/csrc/online-recognizer.h b/sherpa-onnx/csrc/online-recognizer.h index 7d71bc7fa..a5e6f9089 100644 --- a/sherpa-onnx/csrc/online-recognizer.h +++ b/sherpa-onnx/csrc/online-recognizer.h @@ -107,7 +107,8 @@ struct OnlineRecognizerConfig { std::string rule_fars; /// used only for modified_beam_search, if hotwords_buf is non-empty, - /// the hotwords will be loaded from the buffered string in prior to the ${hotwords_file} + /// the hotwords will be loaded from the buffered string in prior to the + /// ${hotwords_file} std::string hotwords_buf; OnlineRecognizerConfig() = default; From ecaba64c74fa6f9ee5e267770ac23b3f98201c30 Mon Sep 17 00:00:00 2001 From: lxiao336 Date: Thu, 12 Sep 2024 16:01:14 +0800 Subject: [PATCH 10/20] Update c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c Co-authored-by: Fangjun Kuang --- .../streaming-zipformer-buffered-tokens-hotwords-c-api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c index c9ee14f49..fa1eaafa0 100644 --- a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c +++ b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c @@ -5,7 +5,7 @@ // // This file demonstrates how to use streaming Zipformer with sherpa-onnx's C -// and with tokens and hotwords loaded from buffered strings instread external +// and with tokens and hotwords loaded from buffered strings instead of from external // files API. // clang-format off // From c212f71b099910b21dd1e9c57d1b2fd5f4c7c06a Mon Sep 17 00:00:00 2001 From: lxiao336 Date: Thu, 12 Sep 2024 16:01:33 +0800 Subject: [PATCH 11/20] Update c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c Co-authored-by: Fangjun Kuang --- .../streaming-zipformer-buffered-tokens-hotwords-c-api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c index fa1eaafa0..a6c8bcc80 100644 --- a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c +++ b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c @@ -30,7 +30,7 @@ size_t read_file(const char *filename, const char **buffer_out) { fseek(file, 0L, SEEK_END); long size = ftell(file); rewind(file); - *buffer_out = malloc(size + 1); + *buffer_out = malloc(size); if (*buffer_out == NULL) { fclose(file); fprintf(stderr, "Memory error\n"); From 12572b44a59c0bcf7bfb7eaba166bdf2686b90bc Mon Sep 17 00:00:00 2001 From: lxiao336 Date: Thu, 12 Sep 2024 16:02:22 +0800 Subject: [PATCH 12/20] Update sherpa-onnx/c-api/c-api.cc Co-authored-by: Fangjun Kuang --- sherpa-onnx/c-api/c-api.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index dfaeaa616..c913cebfc 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -77,8 +77,6 @@ SherpaOnnxOnlineRecognizer *SherpaOnnxCreateOnlineRecognizer( config->model_config.tokens_buf_size > 0) { recognizer_config.model_config.tokens_buf = std::string( config->model_config.tokens_buf, config->model_config.tokens_buf_size); - } else { - recognizer_config.model_config.tokens_buf = ""; } recognizer_config.model_config.num_threads = From 058696ba747a1f51ff3cee7001f86dbefdde0897 Mon Sep 17 00:00:00 2001 From: lxiao336 Date: Thu, 12 Sep 2024 16:02:31 +0800 Subject: [PATCH 13/20] Update c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c Co-authored-by: Fangjun Kuang --- .../streaming-zipformer-buffered-tokens-hotwords-c-api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c index a6c8bcc80..e27a4ad74 100644 --- a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c +++ b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c @@ -21,7 +21,7 @@ #include "sherpa-onnx/c-api/c-api.h" -size_t read_file(const char *filename, const char **buffer_out) { +static size_t ReadFile(const char *filename, const char **buffer_out) { FILE *file = fopen(filename, "rb"); if (file == NULL) { fprintf(stderr, "Failed to open %s\n", filename); From 08d59ff7119f90b8c7cc421200983d1b90685fd9 Mon Sep 17 00:00:00 2001 From: lxiao336 Date: Thu, 12 Sep 2024 16:05:32 +0800 Subject: [PATCH 14/20] Update sherpa-onnx/csrc/online-recognizer-transducer-impl.h Co-authored-by: Fangjun Kuang --- sherpa-onnx/csrc/online-recognizer-transducer-impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h index d1ce240bf..24776383d 100644 --- a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h @@ -85,7 +85,7 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { model_(OnlineTransducerModel::Create(config.model_config)), endpoint_(config_.endpoint_config) { if (!config.model_config.tokens_buf.empty()) { - sym_ = std::move(SymbolTable(config.model_config.tokens_buf, false)); + sym_ = SymbolTable(config.model_config.tokens_buf, false); } else { /// assuming tokens_buf and tokens are guaranteed not being both empty sym_ = std::move(SymbolTable(config.model_config.tokens, true)); From 353b02589b2d1290feddaca790ad9f8670868d60 Mon Sep 17 00:00:00 2001 From: lxiao336 Date: Thu, 12 Sep 2024 16:07:00 +0800 Subject: [PATCH 15/20] Update sherpa-onnx/csrc/online-recognizer.h Co-authored-by: Fangjun Kuang --- sherpa-onnx/csrc/online-recognizer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherpa-onnx/csrc/online-recognizer.h b/sherpa-onnx/csrc/online-recognizer.h index a5e6f9089..eedd30b21 100644 --- a/sherpa-onnx/csrc/online-recognizer.h +++ b/sherpa-onnx/csrc/online-recognizer.h @@ -107,7 +107,7 @@ struct OnlineRecognizerConfig { std::string rule_fars; /// used only for modified_beam_search, if hotwords_buf is non-empty, - /// the hotwords will be loaded from the buffered string in prior to the + /// the hotwords will be loaded from the buffered string instead of from /// ${hotwords_file} std::string hotwords_buf; From 71cd918a0e771be87893ca13bf152f947beeaa2e Mon Sep 17 00:00:00 2001 From: lxiao336 Date: Thu, 12 Sep 2024 16:07:26 +0800 Subject: [PATCH 16/20] Update sherpa-onnx/csrc/online-recognizer-transducer-impl.h Co-authored-by: Fangjun Kuang --- sherpa-onnx/csrc/online-recognizer-transducer-impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h index 24776383d..0bdb1cca4 100644 --- a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h @@ -88,7 +88,7 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { sym_ = SymbolTable(config.model_config.tokens_buf, false); } else { /// assuming tokens_buf and tokens are guaranteed not being both empty - sym_ = std::move(SymbolTable(config.model_config.tokens, true)); + sym_ = SymbolTable(config.model_config.tokens, true); } if (sym_.Contains("")) { From 51b6cc6e164ddab0bbe7b644cc2cc04ec1784a29 Mon Sep 17 00:00:00 2001 From: lxiao336 Date: Thu, 12 Sep 2024 16:07:42 +0800 Subject: [PATCH 17/20] Update sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h Co-authored-by: Fangjun Kuang --- sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h b/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h index 866d7cc2b..c406da5ed 100644 --- a/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h @@ -48,8 +48,7 @@ class OnlineRecognizerTransducerNeMoImpl : public OnlineRecognizerImpl { model_( std::make_unique(config.model_config)) { if (!config.model_config.tokens_buf.empty()) { - symbol_table_ = - std::move(SymbolTable(config.model_config.tokens_buf, false)); + symbol_table_ = SymbolTable(config.model_config.tokens_buf, false); } else { /// assuming tokens_buf and tokens are guaranteed not being both empty symbol_table_ = std::move(SymbolTable(config.model_config.tokens, true)); From 719290b72c7bcf7aa70cb35b5ead8bdb31a991e2 Mon Sep 17 00:00:00 2001 From: lxiao336 Date: Thu, 12 Sep 2024 16:07:52 +0800 Subject: [PATCH 18/20] Update sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h Co-authored-by: Fangjun Kuang --- sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h b/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h index c406da5ed..0a09fdb01 100644 --- a/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h @@ -51,7 +51,7 @@ class OnlineRecognizerTransducerNeMoImpl : public OnlineRecognizerImpl { symbol_table_ = SymbolTable(config.model_config.tokens_buf, false); } else { /// assuming tokens_buf and tokens are guaranteed not being both empty - symbol_table_ = std::move(SymbolTable(config.model_config.tokens, true)); + symbol_table_ = SymbolTable(config.model_config.tokens, true); } if (config.decoding_method == "greedy_search") { From 98fe5e1cd972358cca1a20e79ef34142f6220e5b Mon Sep 17 00:00:00 2001 From: xiao Date: Thu, 12 Sep 2024 16:28:13 +0800 Subject: [PATCH 19/20] for the file "streaming-zipformer-buffered-tokens-hotwords-c-api.c" 1. tokens and hotwords buffers are freed once the recognizier creation is called 2. used ReadFile instead of read_file 3. fixed some other minor suggestions --- .../streaming-zipformer-buffered-tokens-hotwords-c-api.c | 9 +++++++-- sherpa-onnx/c-api/c-api.cc | 2 -- sherpa-onnx/csrc/online-model-config.h | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c index e27a4ad74..0da5f3317 100644 --- a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c +++ b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c @@ -77,14 +77,14 @@ int32_t main() { // reading tokens and hotwords to buffers const char *tokens_buf; - size_t token_buf_size = read_file(tokens_filename, &tokens_buf); + size_t token_buf_size = ReadFile(tokens_filename, &tokens_buf); if (token_buf_size < 1) { fprintf(stderr, "Please check your tokens.txt!\n"); free(tokens_buf); return -1; } const char *hotwords_buf; - size_t hotwords_buf_size = read_file(hotwords_filename, &hotwords_buf); + size_t hotwords_buf_size = ReadFile(hotwords_filename, &hotwords_buf); if (hotwords_buf_size < 1) { fprintf(stderr, "Please check your hotwords.txt!\n"); free(hotwords_buf); @@ -119,6 +119,11 @@ int32_t main() { SherpaOnnxOnlineRecognizer *recognizer = SherpaOnnxCreateOnlineRecognizer(&recognizer_config); + free(tokens_buf); + tokens_buf = NULL; + free(hotwords_buf); + hotwords_buf = NULL; + if (recognizer == NULL) { fprintf(stderr, "Please check your config!\n"); SherpaOnnxFreeWave(wave); diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index c913cebfc..0a4c683ee 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -129,8 +129,6 @@ SherpaOnnxOnlineRecognizer *SherpaOnnxCreateOnlineRecognizer( if (config->hotwords_buf && config->hotwords_buf_size > 0) { recognizer_config.hotwords_buf = std::string(config->hotwords_buf, config->hotwords_buf_size); - } else { - recognizer_config.hotwords_buf = ""; } recognizer_config.blank_penalty = config->blank_penalty; diff --git a/sherpa-onnx/csrc/online-model-config.h b/sherpa-onnx/csrc/online-model-config.h index 127b8d454..a2aaae038 100644 --- a/sherpa-onnx/csrc/online-model-config.h +++ b/sherpa-onnx/csrc/online-model-config.h @@ -46,7 +46,7 @@ struct OnlineModelConfig { std::string bpe_vocab; /// if tokens_buf is non-empty, - /// the tokens will be loaded from the buffered string in prior to the + /// the tokens will be loaded from the buffered string instead of from the /// ${tokens} file std::string tokens_buf; From 1cc767ad0bca031a43cc413a2e992c58b8c98ae5 Mon Sep 17 00:00:00 2001 From: lxiao336 Date: Thu, 12 Sep 2024 16:32:26 +0800 Subject: [PATCH 20/20] Update sherpa-onnx/csrc/online-model-config.cc Co-authored-by: Fangjun Kuang --- sherpa-onnx/csrc/online-model-config.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherpa-onnx/csrc/online-model-config.cc b/sherpa-onnx/csrc/online-model-config.cc index efb5526dd..5592c8d0a 100644 --- a/sherpa-onnx/csrc/online-model-config.cc +++ b/sherpa-onnx/csrc/online-model-config.cc @@ -59,7 +59,7 @@ bool OnlineModelConfig::Validate() const { if (!tokens_buf.empty() && FileExists(tokens)) { SHERPA_ONNX_LOGE( "you can not provide a tokens_buf and a tokens file: '%s', " - "at the same, which is confusing", + "at the same time, which is confusing", tokens.c_str()); return false; }