No public description

PiperOrigin-RevId: 956702795

Author: kylebgorman

mozolm/models/BUILD.bazel

@@ -37,13 +37,13 @@ cc_library(
         ":lm_scores_cc_proto",
         ":model_config_cc_proto",
         "//mozolm/stubs:integral_types",
+        "//third_party/opengrm/sfst",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/status:statusor",
         "@com_google_nisaba//nisaba/port:status_macros",
         "@com_google_nisaba//nisaba/port:utf8_util",
         "@com_google_protobuf//:protobuf",
-        "@org_opengrm_ngram//:opengrm-ngram-lib",
     ],
 )
 
@@ -74,12 +74,12 @@ cc_library(
         ":lm_scores_cc_proto",
         ":model_storage_cc_proto",
         "//mozolm/stubs:integral_types",
+        "//third_party/opengrm/sfst",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/status:statusor",
         "@com_google_absl//absl/strings",
         "@com_google_nisaba//nisaba/port:utf8_util",
         "@com_google_protobuf//:protobuf",
-        "@org_opengrm_ngram//:opengrm-ngram-lib",
     ],
 )
 
@@ -251,6 +251,7 @@ cc_library(
         ":ngram_fst_model",
         ":ngram_word_fst_options_cc_proto",
         "//mozolm/stubs:integral_types",
+        "//third_party/opengrm/sfst",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/status",
@@ -260,7 +261,6 @@ cc_library(
         "@com_google_protobuf//:protobuf",
         "@org_openfst//:fst",
         "@org_openfst//:symbol-table",
-        "@org_opengrm_ngram//:opengrm-ngram-lib",
     ],
 )
 
@@ -334,6 +334,8 @@ cc_library(
         ":model_storage_cc_proto",
         ":ppm_as_fst_options_cc_proto",
         "//mozolm/stubs:integral_types",
+        "//third_party/opengrm/sfst",
+        "//third_party/opengrm/sfst:ngram-count",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/status:statusor",
@@ -345,7 +347,6 @@ cc_library(
         "@com_google_protobuf//:protobuf",
         "@org_openfst//:fst",
         "@org_openfst//:symbol-table",
-        "@org_opengrm_ngram//:opengrm-ngram-lib",
     ],
 )
 

mozolm/models/language_model.cc

@@ -18,7 +18,7 @@
 
 #include "absl/strings/str_cat.h"
 #include "nisaba/port/utf8_util.h"
-#include "ngram/ngram-model.h"
+#include "third_party/opengrm/sfst/sfst.h"
 
 namespace mozolm {
 namespace models {
@@ -75,7 +75,7 @@ void SoftmaxRenormalize(std::vector<double> *neg_log_probs) {
   double kahan_factor = 0.0;
   for (int i = 1; i < neg_log_probs->size(); ++i) {
     tot_prob =
-        ngram::NegLogSum(tot_prob, (*neg_log_probs)[i], &kahan_factor);
+        sfst::NegLogSum(tot_prob, (*neg_log_probs)[i], &kahan_factor);
   }
   for (int i = 0; i < neg_log_probs->size(); ++i) {
     (*neg_log_probs)[i] -= tot_prob;

mozolm/models/language_model_hub.cc

@@ -19,7 +19,7 @@
 
 #include "google/protobuf/stubs/logging.h"
 #include "nisaba/port/utf8_util.h"
-#include "ngram/ngram-model.h"
+#include "third_party/opengrm/sfst/sfst.h"
 #include "nisaba/port/status_macros.h"
 
 namespace mozolm {
@@ -38,7 +38,7 @@ double MixResults(const LMScores& lm_scores, double mix_weight,
     const std::string key = lm_scores.symbols(i);
     double value = -std::log(lm_scores.probabilities(i)) + mix_weight;
     if (mixed_values->contains(key)) {
-      value = ngram::NegLogSum(value, mixed_values->find(key)->second);
+      value = sfst::NegLogSum(value, mixed_values->find(key)->second);
     }
     mixed_values->insert_or_assign(key, value);
   }
@@ -56,7 +56,7 @@ void ExtractMixture(
     if (idx == 0) {
       norm = mixed_value.second;
     } else {
-      norm = ngram::NegLogSum(norm, mixed_value.second);
+      norm = sfst::NegLogSum(norm, mixed_value.second);
     }
     values[idx++] = std::make_pair(mixed_value.first, mixed_value.second);
   }
@@ -96,7 +96,7 @@ absl::Status LanguageModelHub::InitializeModels(const ModelHubConfig& config) {
           // Stores negative log mixing weights for each model.
           mixture_weights_[idx] = config.model_config(idx).weight();
           normalization = (idx == 0) ? mixture_weights_[idx]
-                                     : ngram::NegLogSum(normalization,
+                                     : sfst::NegLogSum(normalization,
                                                         mixture_weights_[idx]);
         }
         for (auto idx = 0; idx < mixture_weights_.size(); ++idx) {
@@ -233,7 +233,7 @@ std::vector<double> LanguageModelHub::GetBayesianMixtureWeights(
     mixture_weights[idx] += bayesian_history_probs_sum[idx];
     normalization = (idx == 0)
                         ? mixture_weights[idx]
-                        : ngram::NegLogSum(normalization, mixture_weights[idx]);
+                        : sfst::NegLogSum(normalization, mixture_weights[idx]);
   }
   for (auto idx = 0; idx < mixture_weights.size(); ++idx) {
     mixture_weights[idx] -= normalization;

mozolm/models/ngram_word_fst_model.cc

@@ -26,7 +26,7 @@
 #include "fst/fst.h"
 #include "fst/matcher.h"
 #include "fst/symbol-table.h"
-#include "ngram/ngram-model.h"
+#include "third_party/opengrm/sfst/sfst.h"
 #include "nisaba/port/status_macros.h"
 
 using nisaba::utf8::DecodeSingleUnicodeChar;
@@ -51,7 +51,7 @@ double SafeNegLogDiff(double cost1, double cost2) {
   if (cost1 >= cost2) {
     return StdArc::Weight::Zero().Value();
   }
-  return ngram::NegLogDiff(cost1, cost2);
+  return sfst::NegLogDiff(cost1, cost2);
 }
 
 // Returns the character at index idx in the unicode string.

mozolm/models/ppm_as_fst_model.cc

@@ -28,8 +28,8 @@
 #include "fst/arcsort.h"
 #include "fst/symbol-table.h"
 #include "fst/vector-fst.h"
-#include "ngram/ngram-count.h"
-#include "ngram/ngram-model.h"
+#include "third_party/opengrm/sfst/ngram-count.h"
+#include "third_party/opengrm/sfst/sfst.h"
 #include "nisaba/port/status_macros.h"
 
 namespace mozolm {
@@ -230,7 +230,7 @@ int IncrementBackoffArcReturnBackoffState(StdVectorFst* fst,
   if (arc.ilabel == 0) {
     backoff_state = arc.nextstate;
     if (increment_count) {
-      arc.weight = StdArc::Weight(ngram::NegLogSum(arc.weight.Value(), 0.0));
+      arc.weight = StdArc::Weight(sfst::NegLogSum(arc.weight.Value(), 0.0));
       arc_iterator.SetValue(arc);
     }
   }
@@ -248,7 +248,7 @@ double GetTotalStateCount(const StdVectorFst& fst, StdArc::StateId s) {
       state_count = arc.weight.Value();
       return state_count;
     }
-    state_count = ngram::NegLogSum(state_count, arc.weight.Value());
+    state_count = sfst::NegLogSum(state_count, arc.weight.Value());
   }
   return state_count;
 }
@@ -271,8 +271,8 @@ absl::StatusOr<double> UpdateIndexProb(double count, double neg_log_beta,
     if (count >= neg_log_beta) {
       return absl::InternalError("Found a count less than \beta.");
     }
-    sym_prob = ngram::NegLogSum(
-        lower_order_prob, ngram::NegLogDiff(count, neg_log_beta) - denominator);
+    sym_prob = sfst::NegLogSum(
+        lower_order_prob, sfst::NegLogDiff(count, neg_log_beta) - denominator);
   }
   return sym_prob;
 }
@@ -356,10 +356,10 @@ absl::Status PpmAsFstModel::AddPriorCounts() {
     StdArc arc = arc_iterator.Value();
     has_unigram.insert(arc.ilabel);
     arc.weight = StdArc::Weight(
-        ngram::NegLogSum(arc.weight.Value(), 0.0));  // Adds 1 count.
+        sfst::NegLogSum(arc.weight.Value(), 0.0));  // Adds 1 count.
     arc_iterator.SetValue(arc);
   }
-  fst_->SetFinal(unigram_state, StdArc::Weight(ngram::NegLogSum(
+  fst_->SetFinal(unigram_state, StdArc::Weight(sfst::NegLogSum(
                                     fst_->Final(unigram_state).Value(), 0.0)));
   bool syms_added = false;
   for (SymbolTableIterator syms_iter(*syms_); !syms_iter.Done();
@@ -474,7 +474,7 @@ absl::Status PpmAsFstModel::Read(const ModelStorage& storage) {
     syms_->AddSymbol("<epsilon>");
     fst_->SetInputSymbols(syms_.get());
     fst_->SetOutputSymbols(syms_.get());
-    ngram_counter_ = std::make_unique<ngram::NGramCounter<Log64Weight>>(
+    ngram_counter_ = std::make_unique<sfst::NGramCounter<Log64Weight>>(
         /*order=*/max_order_);
     if (!storage.model_file().empty()) {
       GOOGLE_LOG(INFO) << "Initializing from training data ...";
@@ -598,7 +598,7 @@ std::vector<double> PpmAsFstModel::InitCacheProbs(
       --num_continuations;
     }
     const double gamma =
-        ngram::NegLogSum(-std::log(num_continuations) - std::log(beta_),
+        sfst::NegLogSum(-std::log(num_continuations) - std::log(beta_),
                          -std::log(alpha_)) - denominator;
     for (size_t i = 0; i < cache_probs.size(); ++i) {
       // Adds in gamma factor to backoff probabilities.
@@ -657,7 +657,7 @@ absl::Status PpmAsFstModel::UpdateCacheAtNonEmptyState(
   std::vector<int> destination_states =
       InitCacheStates(s, backoff_state, backoff_cache, /*arc_origin=*/false);
   const double denominator =
-      ngram::NegLogSum(impl::GetTotalStateCount(*fst_, s), -std::log(alpha_));
+      sfst::NegLogSum(impl::GetTotalStateCount(*fst_, s), -std::log(alpha_));
   std::vector<double> neg_log_probabilities =
       InitCacheProbs(s, backoff_state, backoff_cache, denominator);
   update_status = UpdateCacheStatesAndProbs(
@@ -831,7 +831,7 @@ absl::Status PpmAsFstModel::UpdateHighestFoundState(StdArc::StateId curr_state,
                                                     int sym_index) {
   if (sym_index == 0) {
     // Adds one to final cost and sets destination state to start state.
-    fst_->SetFinal(curr_state, StdArc::Weight(ngram::NegLogSum(
+    fst_->SetFinal(curr_state, StdArc::Weight(sfst::NegLogSum(
                                    fst_->Final(curr_state).Value(), 0.0)));
   } else {
     // Arc with sym_index found at current state.
@@ -853,7 +853,7 @@ absl::Status PpmAsFstModel::UpdateHighestFoundState(StdArc::StateId curr_state,
           new_next_state = state_orders_.size();
           arc.nextstate = new_next_state;
         }
-        arc.weight = StdArc::Weight(ngram::NegLogSum(arc.weight.Value(), 0.0));
+        arc.weight = StdArc::Weight(sfst::NegLogSum(arc.weight.Value(), 0.0));
         arc_iterator.SetValue(arc);
         break;
       }

mozolm/models/ppm_as_fst_model.h

@@ -72,7 +72,7 @@
 #include "mozolm/models/ppm_as_fst_options.pb.h"
 #include "fst/symbol-table.h"
 #include "fst/vector-fst.h"
-#include "ngram/ngram-count.h"
+#include "third_party/opengrm/sfst/ngram-count.h"
 
 namespace mozolm {
 namespace models {
@@ -318,7 +318,7 @@ class PpmAsFstModel : public LanguageModel {
   std::vector<int> state_orders_;  // Stores the order of each state.
   std::unique_ptr<fst::StdVectorFst> fst_;  // Model (counts) stored in FST.
   // For counting character n-grams if training from text file.
-  std::unique_ptr<ngram::NGramCounter<fst::Log64Weight>> ngram_counter_;
+  std::unique_ptr<sfst::NGramCounter<fst::Log64Weight>> ngram_counter_;
   std::unique_ptr<fst::SymbolTable> syms_;  // Character symbols.
 
   // For caching probabilities and destination states for quick access.