77 tprintf(
"Reading ambiguities\n");
80 int test_ambig_part_size;
81 int replacement_ambig_part_size;
85 char *buffer =
new char[kBufferSize];
93 ASSERT_HOST(ambig_file->FGets(buffer, kBufferSize) !=
nullptr && buffer[0] !=
'\0');
95 version =
static_cast<int>(strtol(buffer + 1,
nullptr, 10));
100 while (ambig_file->FGets(buffer, kBufferSize) !=
nullptr) {
102 if (debug_level > 2) {
103 tprintf(
"read line %s\n", buffer);
106 if (!ParseAmbiguityLine(line_num, version, debug_level, encoder_set, buffer,
107 &test_ambig_part_size, test_unichar_ids, &replacement_ambig_part_size,
108 replacement_string, &type)) {
112 auto *ambig_spec =
new AmbigSpec();
113 if (!InsertIntoTable((type ==
REPLACE_AMBIG) ? replace_ambigs_ : dang_ambigs_,
114 test_ambig_part_size, test_unichar_ids, replacement_ambig_part_size,
115 replacement_string, type, ambig_spec, unicharset)) {
120 if (test_ambig_part_size == 1 && replacement_ambig_part_size == 1 && type ==
DEFINITE_AMBIG) {
121 if (one_to_one_definite_ambigs_[test_unichar_ids[0]] ==
nullptr) {
122 one_to_one_definite_ambigs_[test_unichar_ids[0]] =
new UnicharIdVector();
124 one_to_one_definite_ambigs_[test_unichar_ids[0]]->push_back(ambig_spec->correct_ngram_id);
127 if (use_ambigs_for_adaption) {
128 std::vector<UNICHAR_ID> encoding;
131 if (unicharset->encode_string(replacement_string,
true, &encoding,
nullptr,
nullptr)) {
132 for (
int i = 0; i < test_ambig_part_size; ++i) {
133 if (ambigs_for_adaption_[test_unichar_ids[i]] ==
nullptr) {
136 adaption_ambigs_entry = ambigs_for_adaption_[test_unichar_ids[i]];
137 for (
int id_to_insert : encoding) {
143 j < adaption_ambigs_entry->size() && (*adaption_ambigs_entry)[j] > id_to_insert;
147 if (j < adaption_ambigs_entry->size()) {
148 if ((*adaption_ambigs_entry)[j] != id_to_insert) {
149 adaption_ambigs_entry->insert(adaption_ambigs_entry->begin() + j, id_to_insert);
152 adaption_ambigs_entry->push_back(id_to_insert);
162 if (use_ambigs_for_adaption) {
163 for (
size_t i = 0; i < ambigs_for_adaption_.size(); ++i) {
164 adaption_ambigs_entry = ambigs_for_adaption_[i];
165 if (adaption_ambigs_entry ==
nullptr) {
168 for (
size_t j = 0; j < adaption_ambigs_entry->size(); ++j) {
169 UNICHAR_ID ambig_id = (*adaption_ambigs_entry)[j];
170 if (reverse_ambigs_for_adaption_[ambig_id] ==
nullptr) {
173 reverse_ambigs_for_adaption_[ambig_id]->push_back(i);
179 if (debug_level > 1) {
180 for (
int tbl = 0; tbl < 2; ++tbl) {
182 for (
size_t i = 0; i < print_table.size(); ++i) {
183 AmbigSpec_LIST *lst = print_table[i];
184 if (lst ==
nullptr) {
188 tprintf(
"%s Ambiguities for %s:\n", (tbl == 0) ?
"Replaceable" :
"Dangerous",
189 unicharset->debug_str(i).c_str());
191 AmbigSpec_IT lst_it(lst);
192 for (lst_it.mark_cycle_pt(); !lst_it.cycled_list(); lst_it.forward()) {
193 AmbigSpec *ambig_spec = lst_it.data();
201 if (use_ambigs_for_adaption) {
202 for (
int vec_id = 0; vec_id < 2; ++vec_id) {
203 const std::vector<UnicharIdVector *> &vec =
204 (vec_id == 0) ? ambigs_for_adaption_ : reverse_ambigs_for_adaption_;
205 for (
size_t i = 0; i < vec.size(); ++i) {
206 adaption_ambigs_entry = vec[i];
207 if (adaption_ambigs_entry !=
nullptr) {
208 tprintf(
"%sAmbigs for adaption for %s:\n", (vec_id == 0) ?
"" :
"Reverse ",
209 unicharset->debug_str(i).c_str());
210 for (
size_t j = 0; j < adaption_ambigs_entry->size(); ++j) {
211 tprintf(
"%s ", unicharset->debug_str((*adaption_ambigs_entry)[j]).c_str());
std::vector< AmbigSpec_LIST * > UnicharAmbigsVector
void tprintf(const char *format,...)
void chomp_string(char *str)
const int kMaxAmbigStringSize
std::vector< UNICHAR_ID > UnicharIdVector
static void print(const UNICHAR_ID array[], const UNICHARSET &unicharset)