From 81b1afdd83b6ca114afc8f243d049b680db1a6d5 Mon Sep 17 00:00:00 2001 From: Mik Wiseman Date: Tue, 20 Oct 2026 09:27:48 +0300 Subject: [PATCH 1/4] parakeet: honor run_params.vocabulary with phrase boosting Parakeet has no prompt slot, so it ignored transcribe_run_params::vocabulary with a WARN. The TDT and RNN-T greedy decoders now take the terms as phrase boosting over an Aho-Corasick automaton of token ids. The model's own argmax decides between blank and a token; only when it emits a token does the vocabulary re-rank the non-blank tokens, so biasing changes which word is written and never whether one is. A term earns no bonus for its first piece, a broken match gives back what it earned, and a completed term keeps its bonus. Token confidences stay the model's own, and with no terms the decoder never calls into the automaton, so output is bit-exact. The bonus defaults to 8.0 and can be overridden with TRANSCRIBE_VOCABULARY_BOOST. Streaming and CTC heads ignore terms for now. Co-Authored-By: Claude Opus 5.5 --- docs/environment-variables.md | 1 + docs/prompting.md | 18 +++++ src/CMakeLists.txt | 1 + src/arch/parakeet/decoder.cpp | 70 ++++++++++++---- src/arch/parakeet/decoder.h | 31 +++++--- src/arch/parakeet/model.cpp | 62 ++++++++++++++- src/arch/parakeet/parakeet.h | 3 + src/transcribe-phrase-boost.cpp | 137 ++++++++++++++++++++++++++++++++ src/transcribe-phrase-boost.h | 70 ++++++++++++++++ tests/CMakeLists.txt | 14 ++++ tests/phrase_boost_unit.cpp | 85 ++++++++++++++++++++ 11 files changed, 464 insertions(+), 28 deletions(-) create mode 100644 src/transcribe-phrase-boost.cpp create mode 100644 src/transcribe-phrase-boost.h create mode 100644 tests/phrase_boost_unit.cpp diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 9fa946e6a..1b51ab048 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -32,6 +32,7 @@ of tests. | `TRANSCRIBE_CONV_DIRECT_DW` / `TRANSCRIBE_CONV_NO_DIRECT_DW` | Force the depthwise-conv dispatch to the direct `conv_2d_dw` path / the im2col path, overriding the per-family backend default. | | `TRANSCRIBE_CONV_DIRECT_PW` / `TRANSCRIBE_CONV_NO_DIRECT_PW` | Force the pointwise-conv dispatch to direct `mul_mat` / im2col, overriding the backend default. | | `TRANSCRIBE_NO_REPETITION_GUARD` | Turn off the stop for greedy decodes that start repeating themselves, and the trim of a repeating tail at a budget stop (see [`input-limits.md`](input-limits.md)). For byte-exact reference parity; a looping decode then runs to its budget and keeps its repeats. Read once per process. | +| `TRANSCRIBE_VOCABULARY_BOOST=` | Logit bonus per matched vocabulary token for the parakeet transducer decoders (default 8.0; see [`prompting.md`](prompting.md)). For tuning against a term-recall set; `0` keeps the automaton but adds nothing. | | `TRANSCRIBE_DUMP_DIR=` | Enable the per-stage tensor dumper; writes `.f32` + `.json` per dumped tensor into ``. The basis for the numerical-comparison harness (`scripts/compare_tensors.py`). | | `TRANSCRIBE_PERF_DEBUG` | Print a per-stage timing breakdown to stderr (DEBUG log) on the families that profile (`cohere`, `granite`, `canary`, `canary_qwen`, `moonshine`, `moonshine_streaming`, `moss`, `qwen3_asr`, `whisper`). For whisper, a value containing `cpu` or `all` additionally prints the CPU sub-section breakdown. | | `TRANSCRIBE_VOXTRAL_REALTIME_STREAM_TIMING` | Print a per-component streaming wall-time breakdown at stream finalize (voxtral_realtime). | diff --git a/docs/prompting.md b/docs/prompting.md index d908e5c2d..91f4d3e46 100644 --- a/docs/prompting.md +++ b/docs/prompting.md @@ -20,10 +20,28 @@ Probe `transcribe_model_supports()` for the matching feature bit. | Canary 180m-flash / 1b-flash / 1b-v2 | | | | yes | | Fun-ASR-Nano | yes | | | | | MOSS-Transcribe-Diarize | yes | | | | +| Parakeet (TDT and RNN-T heads) | yes | | | | Per-model formats and restrictions are in each model doc's **Prompting:** note under [`models/`](models/). +## Parakeet vocabulary + +Parakeet has no prompt slot, so its transducer decoders take vocabulary terms +as phrase boosting: each term is tokenized the way the model writes it, starting +at a word boundary, and the terms form an Aho-Corasick automaton over token +ids. The model alone decides between blank and a token on every greedy step. +When it emits a token, the vocabulary re-ranks the non-blank tokens. Once the +model has begun a term on its own, each further piece that extends the match +earns a logit bonus (8.0 by default, chosen on a held-out term set), a token +that breaks the match gives back what the unfinished match earned, and a +completed term keeps its bonus. Biasing therefore changes which word is +written, never whether one is, so it cannot invent speech on silence. The +reported token confidences stay the model's own, and with no terms decoding is +bit-exact. Terms with characters the tokenizer cannot write are skipped with a +WARN. Offline runs and batches take terms; streaming ignores them, and CTC +heads do not take them yet. + ## Limits - Over the prompt budget, `vocabulary` drops terms from the end of the list and diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 355c08f6a..1e414edc1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -24,6 +24,7 @@ add_library(transcribe transcribe-model.cpp transcribe-tokenizer.cpp transcribe-prompting.cpp + transcribe-phrase-boost.cpp transcribe-unicode.cpp transcribe-unicode-data.cpp transcribe-debug.cpp diff --git a/src/arch/parakeet/decoder.cpp b/src/arch/parakeet/decoder.cpp index 52defc38f..afd917346 100644 --- a/src/arch/parakeet/decoder.cpp +++ b/src/arch/parakeet/decoder.cpp @@ -13,6 +13,8 @@ #include "decoder.h" +#include + #include "parakeet.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" @@ -966,6 +968,31 @@ int argmax_range(const float * data, int n) { return best_i; } +// Token argmax, biased toward the vocabulary when there is one. +// +// The model alone decides between blank and a token: only when its own +// argmax emits something does the vocabulary re-rank the non-blank tokens. +// Biasing can then change which word is written but never whether one is, +// so it cannot invent speech on silence or noise. The model's own logits are +// left untouched for the confidence that follows. +static int biased_argmax(const float * token_logits, + int n_token_cls, + int blank_id, + const transcribe::PhraseBoost * boost, + int boost_state, + std::vector & scratch) { + const int unbiased = argmax_range(token_logits, n_token_cls); + if (boost == nullptr || boost->empty() || unbiased == blank_id) { + return unbiased; + } + scratch.assign(token_logits, token_logits + n_token_cls); + boost->apply(boost_state, scratch.data(), n_token_cls, blank_id); + if (blank_id >= 0 && blank_id < n_token_cls) { + scratch[static_cast(blank_id)] = -std::numeric_limits::infinity(); + } + return argmax_range(scratch.data(), n_token_cls); +} + // Entropy-based confidence over a token-logit slice. Mirrors the // reference ParakeetTDT.decode_greedy path: // @@ -1015,12 +1042,13 @@ float token_confidence(const float * token_logits, int n_token_classes, std::vec // Greedy decode driver // --------------------------------------------------------------------------- -transcribe_status decode_tdt_greedy(const HostDecoderWeights & w, - const float * enc_out, - int T_enc, - int d_enc, - int n_threads, - std::vector & out_tokens) { +transcribe_status decode_tdt_greedy(const HostDecoderWeights & w, + const float * enc_out, + int T_enc, + int d_enc, + int n_threads, + std::vector & out_tokens, + const transcribe::PhraseBoost * boost) { if (enc_out == nullptr || T_enc <= 0 || d_enc <= 0) { return TRANSCRIBE_ERR_INVALID_ARG; } @@ -1097,6 +1125,10 @@ transcribe_status decode_tdt_greedy(const HostDecoderWeights & w, // the unconditional path. bool predictor_dirty = true; + // Vocabulary biasing state; untouched when no terms were given. + int boost_state = transcribe::PhraseBoost::root; + std::vector boosted; + // See ParakeetHParams::tdt_global_symbol_budget. const bool global_budget = w.tdt_global_symbol_budget; int64_t steps_remaining = static_cast(w.tdt_max_symbols) * T_enc; @@ -1131,7 +1163,7 @@ transcribe_status decode_tdt_greedy(const HostDecoderWeights & w, const float * token_logits = logits.data(); const float * duration_logits = logits.data() + n_token_cls; - const int pred_token = argmax_range(token_logits, n_token_cls); + const int pred_token = biased_argmax(token_logits, n_token_cls, blank_id, boost, boost_state, boosted); const int decision = argmax_range(duration_logits, n_dur); const int duration = w.tdt_durations[static_cast(decision)]; @@ -1172,6 +1204,9 @@ transcribe_status decode_tdt_greedy(const HostDecoderWeights & w, last_token = pred_token; std::swap(state, next_state); // commit predictor_dirty = true; + if (boost != nullptr) { + boost_state = boost->next(boost_state, pred_token); + } } if (global_budget) { @@ -1249,12 +1284,13 @@ transcribe_status decode_tdt_greedy(const HostDecoderWeights & w, // duration). Matches the reference dump points (`dec.embed.0`, // `dec.lstm.{layer}.{h,c}.0`, `dec.joint.0`) emitted on iter 1. -transcribe_status decode_rnnt_greedy(const HostDecoderWeights & w, - const float * enc_out, - int T_enc, - int d_enc, - int n_threads, - std::vector & out_tokens) { +transcribe_status decode_rnnt_greedy(const HostDecoderWeights & w, + const float * enc_out, + int T_enc, + int d_enc, + int n_threads, + std::vector & out_tokens, + const transcribe::PhraseBoost * boost) { if (enc_out == nullptr || T_enc <= 0 || d_enc <= 0) { return TRANSCRIBE_ERR_INVALID_ARG; } @@ -1316,6 +1352,9 @@ transcribe_status decode_rnnt_greedy(const HostDecoderWeights & w, // RNN-T iters/token ratio. bool predictor_dirty = true; + int boost_state = transcribe::PhraseBoost::root; + std::vector boosted; + while (step < T_enc && iter < max_iters) { ++iter; @@ -1342,7 +1381,7 @@ transcribe_status decode_rnnt_greedy(const HostDecoderWeights & w, // RNNT joint output is just `n_token_cls` floats (no duration extras). const float * token_logits = logits.data(); - const int pred_token = argmax_range(token_logits, n_token_cls); + const int pred_token = biased_argmax(token_logits, n_token_cls, blank_id, boost, boost_state, boosted); if (iter == 1 && transcribe::debug::enabled()) { const long long s_h = H; @@ -1377,6 +1416,9 @@ transcribe_status decode_rnnt_greedy(const HostDecoderWeights & w, last_token = pred_token; std::swap(state, next_state); predictor_dirty = true; + if (boost != nullptr) { + boost_state = boost->next(boost_state, pred_token); + } new_symbols += 1; if (w.tdt_max_symbols > 0 && new_symbols >= w.tdt_max_symbols) { diff --git a/src/arch/parakeet/decoder.h b/src/arch/parakeet/decoder.h index e7cb70a55..ff2948d6c 100644 --- a/src/arch/parakeet/decoder.h +++ b/src/arch/parakeet/decoder.h @@ -30,6 +30,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe.h" +#include "transcribe-phrase-boost.h" #include #include @@ -218,24 +219,30 @@ struct TdtToken { // // Returns TRANSCRIBE_OK on success; cannot fail except via invalid args // (validated by the family driver before this is called). -transcribe_status decode_tdt_greedy(const HostDecoderWeights & w, - const float * enc_out, - int T_enc, - int d_enc, - int n_threads, - std::vector & out_tokens); +// +// `boost`, when non-null and non-empty, biases the token argmax toward the +// caller's vocabulary terms (transcribe_run_params::vocabulary); see +// transcribe-phrase-boost.h. Token confidences stay the model's own. +transcribe_status decode_tdt_greedy(const HostDecoderWeights & w, + const float * enc_out, + int T_enc, + int d_enc, + int n_threads, + std::vector & out_tokens, + const transcribe::PhraseBoost * boost = nullptr); // Run RNNT greedy decode end-to-end. Same predictor + joint code as TDT, // but the joint emits `vocab+1` logits (no duration extras) and the step // rule is "blank → advance one frame, non-blank → emit + stay, capped by // tdt_max_symbols". Per-emit duration_frames is fixed at 1. Same I/O // contract and TdtToken result type as decode_tdt_greedy. -transcribe_status decode_rnnt_greedy(const HostDecoderWeights & w, - const float * enc_out, - int T_enc, - int d_enc, - int n_threads, - std::vector & out_tokens); +transcribe_status decode_rnnt_greedy(const HostDecoderWeights & w, + const float * enc_out, + int T_enc, + int d_enc, + int n_threads, + std::vector & out_tokens, + const transcribe::PhraseBoost * boost = nullptr); // Streaming variant of RNN-T greedy decode. Consumes T_enc_new encoder // frames (the chunk just produced) and APPENDS emitted tokens to diff --git a/src/arch/parakeet/model.cpp b/src/arch/parakeet/model.cpp index 81be6f206..76f85a35b 100644 --- a/src/arch/parakeet/model.cpp +++ b/src/arch/parakeet/model.cpp @@ -28,6 +28,7 @@ #include "transcribe-log.h" #include "transcribe-mel.h" #include "transcribe-meta.h" +#include "transcribe-prompting.h" #include "transcribe/parakeet.h" #include "weights.h" @@ -40,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -587,6 +589,10 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par return st; } + // Vocabulary terms bias the greedy transducer decoders (TDT and RNN-T); + // CTC heads do not take them yet. + transcribe::set_feature(m.get(), TRANSCRIBE_FEATURE_VOCABULARY, m->host_decoder.head_kind != HostHeadKind::CTC); + // Multitalker bundle post-load: fuse the diarizer conformer's BN (needs // uploaded data), build its ceil-framed mel frontend, and publish the // DIARIZATION capability — for a bundle it is a property of the file. @@ -789,6 +795,56 @@ static double parakeet_ms_per_enc_frame(const ParakeetHParams & hp) { static_cast(hp.fe_sample_rate); } +// Logit bonus per matched vocabulary token, in natural-log units. Chosen on +// the dev half of a term-recall set and checked against general WER; see +// docs/prompting.md. TRANSCRIBE_VOCABULARY_BOOST overrides it for research. +constexpr float kVocabularyBoost = 8.0f; + +static float vocabulary_boost_score() { + if (const char * v = transcribe::env::str("TRANSCRIBE_VOCABULARY_BOOST")) { + char * end = nullptr; + const float value = std::strtof(v, &end); + if (end != v && value >= 0.0f) { + return value; + } + } + return kVocabularyBoost; +} + +// The phrase-boost automaton for this run's vocabulary, or nullptr when the +// run has none. Terms are tokenized as the model writes them (with the word +// boundary marker), so a term matches only from the start of a word. +static const transcribe::PhraseBoost * vocabulary_boost(ParakeetSession * pc, + const ParakeetModel * pm, + const transcribe_run_params * params) { + std::vector terms = transcribe::prompting::terms(params); + if (terms.empty() || pm->host_decoder.head_kind == HostHeadKind::CTC) { + return nullptr; + } + if (terms != pc->boost_terms || pc->boost.empty()) { + std::vector> phrases; + for (const std::string & term : terms) { + std::vector ids; + if (pm->tok.encode_sentencepiece_bpe(term, ids, 0, -1, /*remove_extra_whitespaces=*/true) != TRANSCRIBE_OK) { + log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "parakeet: this tokenizer cannot encode vocabulary terms; ignoring them"); + return nullptr; + } + bool unknown = false; + for (const int32_t id : ids) { + unknown = unknown || pm->tok.is_unknown(id); + } + if (unknown) { + log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "parakeet: a vocabulary term has characters the model cannot write; skipped"); + continue; + } + phrases.push_back(std::move(ids)); + } + pc->boost.build(phrases, vocabulary_boost_score()); + pc->boost_terms = std::move(terms); + } + return pc->boost.empty() ? nullptr : &pc->boost; +} + // Host-side TDT/RNNT/CTC decode + public result-hierarchy build for one // utterance's encoder output. `enc` is the row-major [T_enc, d_enc] // activation for ONE utterance; utt_index >= 0 tags the per-utterance @@ -827,10 +883,12 @@ static transcribe_status decode_and_populate(ParakeetSession * pc, transcribe_status st = TRANSCRIBE_OK; switch (pm->host_decoder.head_kind) { case HostHeadKind::TDT: - st = decode_tdt_greedy(pm->host_decoder, enc, T_enc, d_enc, pc->n_threads, pc->raw_tokens); + st = decode_tdt_greedy(pm->host_decoder, enc, T_enc, d_enc, pc->n_threads, pc->raw_tokens, + vocabulary_boost(pc, pm, params)); break; case HostHeadKind::RNNT: - st = decode_rnnt_greedy(pm->host_decoder, enc, T_enc, d_enc, pc->n_threads, pc->raw_tokens); + st = decode_rnnt_greedy(pm->host_decoder, enc, T_enc, d_enc, pc->n_threads, pc->raw_tokens, + vocabulary_boost(pc, pm, params)); break; case HostHeadKind::CTC: st = decode_ctc_greedy(pm->host_decoder, enc, T_enc, d_enc, pc->n_threads, pc->raw_tokens); diff --git a/src/arch/parakeet/parakeet.h b/src/arch/parakeet/parakeet.h index 30ae27439..719d99691 100644 --- a/src/arch/parakeet/parakeet.h +++ b/src/arch/parakeet/parakeet.h @@ -318,6 +318,9 @@ struct ParakeetSession final : public transcribe_session { std::vector pos_div_term; std::vector enc_host; std::vector raw_tokens; + // Vocabulary biasing, rebuilt only when the run's terms change. + std::vector boost_terms; + transcribe::PhraseBoost boost; // Multitalker bundle scratch: streaming-diarizer AOSC/FIFO state for // the embedded sortformer forward (diarize=ON runs only). diff --git a/src/transcribe-phrase-boost.cpp b/src/transcribe-phrase-boost.cpp new file mode 100644 index 000000000..db4f29b74 --- /dev/null +++ b/src/transcribe-phrase-boost.cpp @@ -0,0 +1,137 @@ +// transcribe-phrase-boost.cpp - see transcribe-phrase-boost.h. + +#include "transcribe-phrase-boost.h" + +#include +#include + +namespace transcribe { + +namespace { + +bool by_token(const PhraseBoost::Edge & e, int32_t token) { + return e.token < token; +} + +} // namespace + +int PhraseBoost::child(int state, int32_t token) const { + const auto & c = nodes_[static_cast(state)].children; + const auto it = std::lower_bound(c.begin(), c.end(), token, + [](const std::pair & p, int32_t t) { return p.first < t; }); + return (it != c.end() && it->first == token) ? it->second : -1; +} + +void PhraseBoost::build(const std::vector> & phrases, float score) { + nodes_.assign(1, Node{}); + edges_.clear(); + score_ = score; + std::vector end(1, false); + + // The trie: one node per distinct token prefix. + for (const std::vector & phrase : phrases) { + if (phrase.empty()) { + continue; + } + int state = root; + for (const int32_t token : phrase) { + int next = child(state, token); + if (next < 0) { + next = static_cast(nodes_.size()); + Node node; + node.earned = nodes_[static_cast(state)].earned + (state == root ? 0.0f : score); + nodes_.push_back(node); + end.push_back(false); + auto & c = nodes_[static_cast(state)].children; + c.insert(std::lower_bound(c.begin(), c.end(), std::make_pair(token, -1)), std::make_pair(token, next)); + } + state = next; + } + end[static_cast(state)] = true; + } + + // Failure links and kept bonuses, breadth first so a node's parent and + // its failure target are always finished before it. + std::deque queue; + for (const auto & [token, c] : nodes_[root].children) { + nodes_[static_cast(c)].fail = root; + nodes_[static_cast(c)].locked = end[static_cast(c)] ? nodes_[static_cast(c)].earned : 0.0f; + queue.push_back(c); + } + while (!queue.empty()) { + const int u = queue.front(); + queue.pop_front(); + for (const auto & [token, v] : nodes_[static_cast(u)].children) { + int f = nodes_[static_cast(u)].fail; + while (f != root && child(f, token) < 0) { + f = nodes_[static_cast(f)].fail; + } + const int target = child(f, token); + nodes_[static_cast(v)].fail = (target >= 0 && target != v) ? target : root; + nodes_[static_cast(v)].locked = end[static_cast(v)] ? nodes_[static_cast(v)].earned : + nodes_[static_cast(u)].locked; + queue.push_back(v); + } + } + + // Every transition out of every state, resolved through the failure + // chain once here instead of on each decode step. The deepest match wins. + edges_.assign(nodes_.size(), {}); + for (size_t s = 0; s < nodes_.size(); ++s) { + const Node & from = nodes_[s]; + std::vector & out = edges_[s]; + for (int f = static_cast(s);; f = nodes_[static_cast(f)].fail) { + for (const auto & [token, target] : nodes_[static_cast(f)].children) { + const auto at = std::lower_bound(out.begin(), out.end(), token, by_token); + if (at != out.end() && at->token == token) { + continue; + } + // Extending the match keeps everything earned so far. Falling + // back gives back the unfinished part and keeps completed + // terms; the suffix that still matches is earned again. + const bool extends = f == static_cast(s); + const float delta = + nodes_[static_cast(target)].earned - from.earned + (extends ? 0.0f : from.locked); + out.insert(at, Edge{ token, target, delta }); + } + if (f == root) { + break; + } + } + } +} + +int PhraseBoost::next(int state, int32_t token) const { + const auto & out = edges_[static_cast(state)]; + const auto at = std::lower_bound(out.begin(), out.end(), token, by_token); + return (at != out.end() && at->token == token) ? at->target : root; +} + +float PhraseBoost::delta(int state, int32_t token) const { + const auto & out = edges_[static_cast(state)]; + const auto at = std::lower_bound(out.begin(), out.end(), token, by_token); + if (at != out.end() && at->token == token) { + return at->delta; + } + const Node & from = nodes_[static_cast(state)]; + return from.locked - from.earned; +} + +void PhraseBoost::apply(int state, float * logits, int n, int32_t blank) const { + const Node & from = nodes_[static_cast(state)]; + const float base = from.locked - from.earned; + if (base != 0.0f) { + for (int t = 0; t < n; ++t) { + if (t != blank) { + logits[t] += base; + } + } + } + for (const Edge & e : edges_[static_cast(state)]) { + if (e.token >= 0 && e.token < n && e.token != blank) { + logits[e.token] += e.delta - base; + } + } +} + +} // namespace transcribe diff --git a/src/transcribe-phrase-boost.h b/src/transcribe-phrase-boost.h new file mode 100644 index 000000000..141514d35 --- /dev/null +++ b/src/transcribe-phrase-boost.h @@ -0,0 +1,70 @@ +// transcribe-phrase-boost.h - token-level phrase boosting for greedy decoders. +// +// The caller's vocabulary terms, tokenized, become an Aho-Corasick automaton +// over token ids. A greedy decoder keeps one automaton state and, before each +// argmax, adds to every candidate token the score that emitting it would +// earn: +// +// - a token that extends the current partial match earns `score`; +// - a token that breaks it gives back what the unfinished match earned and +// may start a new match through the failure links; +// - a completed term keeps its bonus, so later tokens never take it back. +// +// The blank token never moves the automaton and is never adjusted. With no +// terms the decoder skips every call, so unbiased decoding stays bit-exact. +// This is shallow fusion with a boosting tree (as in icefall's ContextGraph +// and NeMo's phrase boosting), reduced to what one greedy hypothesis needs. + +#pragma once + +#include +#include +#include + +namespace transcribe { + +class PhraseBoost { + public: + // Builds the automaton. Empty phrases are ignored; duplicates merge. + // `score` is in logit units (natural log), applied per matched token after + // a term's first one: the first piece is often a short, common word start, + // and a bonus there rewrites ordinary words. A term is completed once the + // model has begun it, and nothing is started for it. + void build(const std::vector> & phrases, float score); + + bool empty() const { return nodes_.size() <= 1; } + + static constexpr int root = 0; + + // The state after emitting `token` from `state`. + int next(int state, int32_t token) const; + + // Adds to logits[0, n) the score each non-blank token would earn from + // `state`. Tokens outside [0, n) in the automaton are ignored. + void apply(int state, float * logits, int n, int32_t blank) const; + + // The score emitting `token` from `state` earns (what apply adds to it). + float delta(int state, int32_t token) const; + + struct Edge { + int32_t token; + int target; + float delta; + }; + + private: + struct Node { + std::vector> children; // sorted by token + int fail = 0; + float earned = 0.0f; // bonus a partial match to here has collected + float locked = 0.0f; // part of `earned` that completed terms keep + }; + + int child(int state, int32_t token) const; + + std::vector nodes_; + std::vector> edges_; // per state, sorted by token + float score_ = 0.0f; +}; + +} // namespace transcribe diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 110645b02..2faf94be4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -295,6 +295,20 @@ transcribe_apply_warnings(transcribe_decode_budget_unit) add_test(NAME transcribe_decode_budget_unit COMMAND transcribe_decode_budget_unit) +# ----------------------------------------------------------------------------- +# Phrase boosting automaton (pure host, no model) + +add_executable(transcribe_phrase_boost_unit + phrase_boost_unit.cpp + ${CMAKE_SOURCE_DIR}/src/transcribe-phrase-boost.cpp) + +target_include_directories(transcribe_phrase_boost_unit PRIVATE + ${CMAKE_SOURCE_DIR}/src) + +transcribe_apply_warnings(transcribe_phrase_boost_unit) + +add_test(NAME transcribe_phrase_boost_unit COMMAND transcribe_phrase_boost_unit) + # ----------------------------------------------------------------------------- # Greedy-decode repetition guard (pure host, no model) diff --git a/tests/phrase_boost_unit.cpp b/tests/phrase_boost_unit.cpp new file mode 100644 index 000000000..98fae5d9a --- /dev/null +++ b/tests/phrase_boost_unit.cpp @@ -0,0 +1,85 @@ +// Phrase boosting automaton (pure host, no model). The scores are the whole +// contract: a wrong one either invents terms or never finds them, and the +// decoder has no other check. + +#include "transcribe-phrase-boost.h" + +#include +#include +#include + +namespace { + +int g_failures = 0; + +void expect(const char * what, float got, float expected) { + if (std::fabs(got - expected) > 1e-6f) { + std::fprintf(stderr, "FAIL %s: got %.3f, expected %.3f\n", what, got, expected); + ++g_failures; + } +} + +// Emits `tokens` from the root and returns the summed score. +float walk(const transcribe::PhraseBoost & b, const std::vector & tokens, int * end_state = nullptr) { + int state = transcribe::PhraseBoost::root; + float total = 0.0f; + for (const int32_t t : tokens) { + total += b.delta(state, t); + state = b.next(state, t); + } + if (end_state != nullptr) { + *end_state = state; + } + return total; +} + +} // namespace + +int main(void) { + transcribe::PhraseBoost b; + b.build({ { 1, 2, 3 }, { 2, 4 }, {}, { 1, 2, 3 } }, 1.0f); + + // A term's first piece earns nothing; each later matched piece earns the + // score; an unrelated token earns nothing. + expect("start a term", b.delta(transcribe::PhraseBoost::root, 1), 0.0f); + expect("continue a term", b.delta(b.next(transcribe::PhraseBoost::root, 1), 2), 1.0f); + expect("unrelated from root", b.delta(transcribe::PhraseBoost::root, 9), 0.0f); + expect("complete term", walk(b, { 1, 2, 3 }), 2.0f); + + // A completed term keeps its bonus whatever follows. + expect("completed then unrelated", walk(b, { 1, 2, 3, 9, 9 }), 2.0f); + + // A broken match gives back exactly what it had earned. + expect("broken match", walk(b, { 1, 2, 9 }), 0.0f); + expect("broken at once", walk(b, { 1, 9 }), 0.0f); + + // Falling back keeps the suffix that still matches another term: + // 1 2 then 4 completes {2, 4} through the failure link. + int state = 0; + expect("suffix completes another term", walk(b, { 1, 2, 4 }, &state), 1.0f); + expect("and keeps it", walk(b, { 1, 2, 4, 9 }), 1.0f); + + // apply() adds exactly delta() to each non-blank token and leaves blank. + int s12 = 0; + walk(b, { 1, 2 }, &s12); + std::vector logits(6, 0.0f); + b.apply(s12, logits.data(), static_cast(logits.size()), /*blank=*/5); + for (int t = 0; t < 5; ++t) { + expect("apply matches delta", logits[static_cast(t)], b.delta(s12, t)); + } + expect("blank untouched", logits[5], 0.0f); + + // No phrases: empty, and every score is zero. + transcribe::PhraseBoost none; + none.build({}, 2.0f); + if (!none.empty()) { + std::fprintf(stderr, "FAIL empty automaton reports terms\n"); + ++g_failures; + } + expect("empty automaton", none.delta(transcribe::PhraseBoost::root, 1), 0.0f); + + if (g_failures == 0) { + std::printf("phrase_boost_unit: ok\n"); + } + return g_failures == 0 ? 0 : 1; +} From 0898790a3eca628264b8f965c3536cf283f99b37 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Fri, 20 Nov 2026 17:16:02 +0800 Subject: [PATCH 2/4] parakeet phrase boost: clang-format --- src/arch/parakeet/decoder.cpp | 4 ++-- src/arch/parakeet/decoder.h | 2 +- src/arch/parakeet/model.cpp | 9 ++++++--- src/arch/parakeet/parakeet.h | 10 +++++----- src/transcribe-phrase-boost.cpp | 10 ++++++---- src/transcribe-phrase-boost.h | 2 +- tests/phrase_boost_unit.cpp | 9 ++++++++- 7 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/arch/parakeet/decoder.cpp b/src/arch/parakeet/decoder.cpp index afd917346..d711bf3f0 100644 --- a/src/arch/parakeet/decoder.cpp +++ b/src/arch/parakeet/decoder.cpp @@ -13,14 +13,14 @@ #include "decoder.h" -#include - #include "parakeet.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" #include "transcribe-log.h" #include "weights.h" +#include + // ggml-backend.h, not ggml-cpu.h: under GGML_BACKEND_DL the CPU backend // is a loadable module, so backend-specific entry points must be reached // through the registry (ggml_backend_init_by_type / get_proc_address), diff --git a/src/arch/parakeet/decoder.h b/src/arch/parakeet/decoder.h index ff2948d6c..b05609c8f 100644 --- a/src/arch/parakeet/decoder.h +++ b/src/arch/parakeet/decoder.h @@ -29,8 +29,8 @@ #include "ggml-backend.h" #include "ggml.h" -#include "transcribe.h" #include "transcribe-phrase-boost.h" +#include "transcribe.h" #include #include diff --git a/src/arch/parakeet/model.cpp b/src/arch/parakeet/model.cpp index 76f85a35b..2882d152c 100644 --- a/src/arch/parakeet/model.cpp +++ b/src/arch/parakeet/model.cpp @@ -825,8 +825,10 @@ static const transcribe::PhraseBoost * vocabulary_boost(ParakeetSession * std::vector> phrases; for (const std::string & term : terms) { std::vector ids; - if (pm->tok.encode_sentencepiece_bpe(term, ids, 0, -1, /*remove_extra_whitespaces=*/true) != TRANSCRIBE_OK) { - log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "parakeet: this tokenizer cannot encode vocabulary terms; ignoring them"); + if (pm->tok.encode_sentencepiece_bpe(term, ids, 0, -1, /*remove_extra_whitespaces=*/true) != + TRANSCRIBE_OK) { + log_msg(TRANSCRIBE_LOG_LEVEL_WARN, + "parakeet: this tokenizer cannot encode vocabulary terms; ignoring them"); return nullptr; } bool unknown = false; @@ -834,7 +836,8 @@ static const transcribe::PhraseBoost * vocabulary_boost(ParakeetSession * unknown = unknown || pm->tok.is_unknown(id); } if (unknown) { - log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "parakeet: a vocabulary term has characters the model cannot write; skipped"); + log_msg(TRANSCRIBE_LOG_LEVEL_WARN, + "parakeet: a vocabulary term has characters the model cannot write; skipped"); continue; } phrases.push_back(std::move(ids)); diff --git a/src/arch/parakeet/parakeet.h b/src/arch/parakeet/parakeet.h index 719d99691..9d2fee81c 100644 --- a/src/arch/parakeet/parakeet.h +++ b/src/arch/parakeet/parakeet.h @@ -313,11 +313,11 @@ struct ParakeetSession final : public transcribe_session { ggml_tensor * encoder_out = nullptr; // Per-context scratch reused across runs. - std::vector mel_buf; - std::vector pos_buf; - std::vector pos_div_term; - std::vector enc_host; - std::vector raw_tokens; + std::vector mel_buf; + std::vector pos_buf; + std::vector pos_div_term; + std::vector enc_host; + std::vector raw_tokens; // Vocabulary biasing, rebuilt only when the run's terms change. std::vector boost_terms; transcribe::PhraseBoost boost; diff --git a/src/transcribe-phrase-boost.cpp b/src/transcribe-phrase-boost.cpp index db4f29b74..c17336591 100644 --- a/src/transcribe-phrase-boost.cpp +++ b/src/transcribe-phrase-boost.cpp @@ -54,8 +54,9 @@ void PhraseBoost::build(const std::vector> & phrases, float // its failure target are always finished before it. std::deque queue; for (const auto & [token, c] : nodes_[root].children) { - nodes_[static_cast(c)].fail = root; - nodes_[static_cast(c)].locked = end[static_cast(c)] ? nodes_[static_cast(c)].earned : 0.0f; + nodes_[static_cast(c)].fail = root; + nodes_[static_cast(c)].locked = + end[static_cast(c)] ? nodes_[static_cast(c)].earned : 0.0f; queue.push_back(c); } while (!queue.empty()) { @@ -68,8 +69,9 @@ void PhraseBoost::build(const std::vector> & phrases, float } const int target = child(f, token); nodes_[static_cast(v)].fail = (target >= 0 && target != v) ? target : root; - nodes_[static_cast(v)].locked = end[static_cast(v)] ? nodes_[static_cast(v)].earned : - nodes_[static_cast(u)].locked; + nodes_[static_cast(v)].locked = end[static_cast(v)] ? + nodes_[static_cast(v)].earned : + nodes_[static_cast(u)].locked; queue.push_back(v); } } diff --git a/src/transcribe-phrase-boost.h b/src/transcribe-phrase-boost.h index 141514d35..b57cfa98d 100644 --- a/src/transcribe-phrase-boost.h +++ b/src/transcribe-phrase-boost.h @@ -54,7 +54,7 @@ class PhraseBoost { private: struct Node { - std::vector> children; // sorted by token + std::vector> children; // sorted by token int fail = 0; float earned = 0.0f; // bonus a partial match to here has collected float locked = 0.0f; // part of `earned` that completed terms keep diff --git a/tests/phrase_boost_unit.cpp b/tests/phrase_boost_unit.cpp index 98fae5d9a..d1ebcfc4b 100644 --- a/tests/phrase_boost_unit.cpp +++ b/tests/phrase_boost_unit.cpp @@ -37,7 +37,14 @@ float walk(const transcribe::PhraseBoost & b, const std::vector & token int main(void) { transcribe::PhraseBoost b; - b.build({ { 1, 2, 3 }, { 2, 4 }, {}, { 1, 2, 3 } }, 1.0f); + b.build( + { + { 1, 2, 3 }, + { 2, 4 }, + {}, + { 1, 2, 3 } + }, + 1.0f); // A term's first piece earns nothing; each later matched piece earns the // score; an unrelated token earns nothing. From 4b72726ce6b76506a78d98997e2558a5fc42f222 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Fri, 20 Nov 2026 17:16:11 +0800 Subject: [PATCH 3/4] parakeet: vocabulary boost 3.0, warn on streaming, name skipped terms Lower the per-token vocabulary bonus from 8.0 to 3.0 and drop the TRANSCRIBE_VOCABULARY_BOOST override. At 8.0 a term whose first piece is short (a bare word-boundary piece, a single letter) rewrote unrelated words, e.g. the term 2FA turned "$2 billion company" into "2 billion dollar" on clean speech. 3.0 kept the term-recall gains on the probe set and left transcripts of unrelated speech unchanged. Streaming does not apply vocabulary terms; warn instead of dropping them silently now that the model advertises the feature. Name the term in the skipped-term warning, and document (and pin in a test) that a term completed only inside a longer unfinished match is not kept. --- docs/environment-variables.md | 1 - docs/prompting.md | 11 ++++++----- src/arch/parakeet/model.cpp | 32 +++++++++++++++----------------- src/transcribe-phrase-boost.h | 3 +++ tests/phrase_boost_unit.cpp | 28 ++++++++++++++++++++-------- 5 files changed, 44 insertions(+), 31 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 1b51ab048..9fa946e6a 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -32,7 +32,6 @@ of tests. | `TRANSCRIBE_CONV_DIRECT_DW` / `TRANSCRIBE_CONV_NO_DIRECT_DW` | Force the depthwise-conv dispatch to the direct `conv_2d_dw` path / the im2col path, overriding the per-family backend default. | | `TRANSCRIBE_CONV_DIRECT_PW` / `TRANSCRIBE_CONV_NO_DIRECT_PW` | Force the pointwise-conv dispatch to direct `mul_mat` / im2col, overriding the backend default. | | `TRANSCRIBE_NO_REPETITION_GUARD` | Turn off the stop for greedy decodes that start repeating themselves, and the trim of a repeating tail at a budget stop (see [`input-limits.md`](input-limits.md)). For byte-exact reference parity; a looping decode then runs to its budget and keeps its repeats. Read once per process. | -| `TRANSCRIBE_VOCABULARY_BOOST=` | Logit bonus per matched vocabulary token for the parakeet transducer decoders (default 8.0; see [`prompting.md`](prompting.md)). For tuning against a term-recall set; `0` keeps the automaton but adds nothing. | | `TRANSCRIBE_DUMP_DIR=` | Enable the per-stage tensor dumper; writes `.f32` + `.json` per dumped tensor into ``. The basis for the numerical-comparison harness (`scripts/compare_tensors.py`). | | `TRANSCRIBE_PERF_DEBUG` | Print a per-stage timing breakdown to stderr (DEBUG log) on the families that profile (`cohere`, `granite`, `canary`, `canary_qwen`, `moonshine`, `moonshine_streaming`, `moss`, `qwen3_asr`, `whisper`). For whisper, a value containing `cpu` or `all` additionally prints the CPU sub-section breakdown. | | `TRANSCRIBE_VOXTRAL_REALTIME_STREAM_TIMING` | Print a per-component streaming wall-time breakdown at stream finalize (voxtral_realtime). | diff --git a/docs/prompting.md b/docs/prompting.md index 91f4d3e46..f6b2cc403 100644 --- a/docs/prompting.md +++ b/docs/prompting.md @@ -33,14 +33,15 @@ at a word boundary, and the terms form an Aho-Corasick automaton over token ids. The model alone decides between blank and a token on every greedy step. When it emits a token, the vocabulary re-ranks the non-blank tokens. Once the model has begun a term on its own, each further piece that extends the match -earns a logit bonus (8.0 by default, chosen on a held-out term set), a token -that breaks the match gives back what the unfinished match earned, and a -completed term keeps its bonus. Biasing therefore changes which word is +earns a logit bonus of 3.0, a token that breaks the match gives back what the +unfinished match earned, and a completed term keeps its bonus. 3.0 is the +lowest bonus that kept the term-recall gains while leaving transcripts of +unrelated speech unchanged. Biasing therefore changes which word is written, never whether one is, so it cannot invent speech on silence. The reported token confidences stay the model's own, and with no terms decoding is bit-exact. Terms with characters the tokenizer cannot write are skipped with a -WARN. Offline runs and batches take terms; streaming ignores them, and CTC -heads do not take them yet. +WARN. Offline runs and batches take terms; streaming ignores them with a WARN, +and CTC heads do not take them yet. ## Limits diff --git a/src/arch/parakeet/model.cpp b/src/arch/parakeet/model.cpp index 2882d152c..775070317 100644 --- a/src/arch/parakeet/model.cpp +++ b/src/arch/parakeet/model.cpp @@ -795,21 +795,10 @@ static double parakeet_ms_per_enc_frame(const ParakeetHParams & hp) { static_cast(hp.fe_sample_rate); } -// Logit bonus per matched vocabulary token, in natural-log units. Chosen on -// the dev half of a term-recall set and checked against general WER; see -// docs/prompting.md. TRANSCRIBE_VOCABULARY_BOOST overrides it for research. -constexpr float kVocabularyBoost = 8.0f; - -static float vocabulary_boost_score() { - if (const char * v = transcribe::env::str("TRANSCRIBE_VOCABULARY_BOOST")) { - char * end = nullptr; - const float value = std::strtof(v, &end); - if (end != v && value >= 0.0f) { - return value; - } - } - return kVocabularyBoost; -} +// Logit bonus per matched vocabulary token after a term's first, in +// natural-log units. The lowest value that kept the term-recall gains while +// leaving transcripts of unrelated speech unchanged; see docs/prompting.md. +constexpr float kVocabularyBoost = 3.0f; // The phrase-boost automaton for this run's vocabulary, or nullptr when the // run has none. Terms are tokenized as the model writes them (with the word @@ -837,12 +826,12 @@ static const transcribe::PhraseBoost * vocabulary_boost(ParakeetSession * } if (unknown) { log_msg(TRANSCRIBE_LOG_LEVEL_WARN, - "parakeet: a vocabulary term has characters the model cannot write; skipped"); + "parakeet: vocabulary term '%s' has characters the model cannot write; skipped", term.c_str()); continue; } phrases.push_back(std::move(ids)); } - pc->boost.build(phrases, vocabulary_boost_score()); + pc->boost.build(phrases, kVocabularyBoost); pc->boost_terms = std::move(terms); } return pc->boost.empty() ? nullptr : &pc->boost; @@ -3467,6 +3456,15 @@ transcribe_status stream_begin(transcribe_session * session, pc->stream_audio_input_samples = 0; + // Vocabulary biasing is wired into the offline decoders only; say so + // rather than drop the terms silently. + if (run_params->n_vocabulary > 0) { + log_msg(TRANSCRIBE_LOG_LEVEL_WARN, + "parakeet: vocabulary terms are not applied on the streaming API yet; ignoring %d term(s) " + "(use transcribe_run)", + run_params->n_vocabulary); + } + // -------- Buffered streaming path (parakeet-unified-en-0.6b) -------- // // chunked_limited_with_rc with a 3-tuple training menu. Re-runs the diff --git a/src/transcribe-phrase-boost.h b/src/transcribe-phrase-boost.h index b57cfa98d..37591179e 100644 --- a/src/transcribe-phrase-boost.h +++ b/src/transcribe-phrase-boost.h @@ -9,6 +9,9 @@ // - a token that breaks it gives back what the unfinished match earned and // may start a new match through the failure links; // - a completed term keeps its bonus, so later tokens never take it back. +// A term completed only as the tail of a longer, still unfinished match +// ("York City" inside "New York City") is not kept when that match +// breaks. // // The blank token never moves the automaton and is never adjusted. With no // terms the decoder skips every call, so unbiased decoding stays bit-exact. diff --git a/tests/phrase_boost_unit.cpp b/tests/phrase_boost_unit.cpp index d1ebcfc4b..62bbf2681 100644 --- a/tests/phrase_boost_unit.cpp +++ b/tests/phrase_boost_unit.cpp @@ -10,6 +10,8 @@ namespace { +using Phrases = std::vector>; + int g_failures = 0; void expect(const char * what, float got, float expected) { @@ -36,15 +38,14 @@ float walk(const transcribe::PhraseBoost & b, const std::vector & token } // namespace int main(void) { + const Phrases terms = { + { 1, 2, 3 }, + { 2, 4 }, + {}, + { 1, 2, 3 } + }; transcribe::PhraseBoost b; - b.build( - { - { 1, 2, 3 }, - { 2, 4 }, - {}, - { 1, 2, 3 } - }, - 1.0f); + b.build(terms, 1.0f); // A term's first piece earns nothing; each later matched piece earns the // score; an unrelated token earns nothing. @@ -66,6 +67,17 @@ int main(void) { expect("suffix completes another term", walk(b, { 1, 2, 4 }, &state), 1.0f); expect("and keeps it", walk(b, { 1, 2, 4, 9 }), 1.0f); + // A term completed only as the tail of a longer unfinished match is not + // kept when that match breaks: {2, 3} inside {1, 2, 3, 4}. + const Phrases tail_terms = { + { 1, 2, 3, 4 }, + { 2, 3 } + }; + transcribe::PhraseBoost tail; + tail.build(tail_terms, 1.0f); + expect("tail term on its own", walk(tail, { 2, 3, 9 }), 1.0f); + expect("tail term inside a broken match", walk(tail, { 1, 2, 3, 9 }), 0.0f); + // apply() adds exactly delta() to each non-blank token and leaves blank. int s12 = 0; walk(b, { 1, 2 }, &s12); From 5146886901139e75a7566dac3c40f475fb627128 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Fri, 20 Nov 2026 17:18:50 +0800 Subject: [PATCH 4/4] parakeet: match vocabulary terms with the first letter's case flipped Sentence position decides the case of a term's first letter in the transcript, so a term typed "claude" missed "Claude", and a lowercase term missed its capitalized form at the start of a sentence. Each term now also matches with its first ASCII letter's case flipped. On the probe set this kept every unrelated transcript unchanged and recovered terms typed in lowercase. --- docs/prompting.md | 4 +++- src/arch/parakeet/model.cpp | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/docs/prompting.md b/docs/prompting.md index f6b2cc403..b33812647 100644 --- a/docs/prompting.md +++ b/docs/prompting.md @@ -36,7 +36,9 @@ model has begun a term on its own, each further piece that extends the match earns a logit bonus of 3.0, a token that breaks the match gives back what the unfinished match earned, and a completed term keeps its bonus. 3.0 is the lowest bonus that kept the term-recall gains while leaving transcripts of -unrelated speech unchanged. Biasing therefore changes which word is +unrelated speech unchanged. Each term also matches with the case of its first +letter flipped, since sentence position decides that letter's case in the +transcript; other letters match as given. Biasing therefore changes which word is written, never whether one is, so it cannot invent speech on silence. The reported token confidences stay the model's own, and with no terms decoding is bit-exact. Terms with characters the tokenizer cannot write are skipped with a diff --git a/src/arch/parakeet/model.cpp b/src/arch/parakeet/model.cpp index 775070317..8ade1bb15 100644 --- a/src/arch/parakeet/model.cpp +++ b/src/arch/parakeet/model.cpp @@ -800,9 +800,29 @@ static double parakeet_ms_per_enc_frame(const ParakeetHParams & hp) { // leaving transcripts of unrelated speech unchanged; see docs/prompting.md. constexpr float kVocabularyBoost = 3.0f; +// `term` with the case of its first letter flipped ("Postgres" -> +// "postgres"), or "" when it does not start with an ASCII letter. Sentence +// position decides that letter's case in the transcript, not the term list. +static std::string flip_first_letter(const std::string & term) { + std::string out = transcribe::prompting::strip(term); + if (out.empty()) { + return ""; + } + const char c = out[0]; + if (c >= 'a' && c <= 'z') { + out[0] = static_cast(c - 'a' + 'A'); + } else if (c >= 'A' && c <= 'Z') { + out[0] = static_cast(c - 'A' + 'a'); + } else { + return ""; + } + return out; +} + // The phrase-boost automaton for this run's vocabulary, or nullptr when the // run has none. Terms are tokenized as the model writes them (with the word -// boundary marker), so a term matches only from the start of a word. +// boundary marker), so a term matches only from the start of a word. Each +// term also matches with its first letter's case flipped. static const transcribe::PhraseBoost * vocabulary_boost(ParakeetSession * pc, const ParakeetModel * pm, const transcribe_run_params * params) { @@ -830,6 +850,19 @@ static const transcribe::PhraseBoost * vocabulary_boost(ParakeetSession * continue; } phrases.push_back(std::move(ids)); + + const std::string flipped = flip_first_letter(term); + if (!flipped.empty() && + pm->tok.encode_sentencepiece_bpe(flipped, ids, 0, -1, + /*remove_extra_whitespaces=*/true) == TRANSCRIBE_OK) { + bool flipped_unknown = false; + for (const int32_t id : ids) { + flipped_unknown = flipped_unknown || pm->tok.is_unknown(id); + } + if (!flipped_unknown) { + phrases.push_back(std::move(ids)); + } + } } pc->boost.build(phrases, kVocabularyBoost); pc->boost_terms = std::move(terms);